@lincoln504/pi-research 1.6.18 → 1.6.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/agent-skill/pi-research/README.md +1 -1
- package/agent-skill/pi-research/SKILL.md +1 -1
- package/dist/cli.mjs +343 -55
- package/dist/thread-worker.mjs +7 -0
- package/docs/AGENT-SKILL.md +1 -1
- package/docs/CHANGELOG.md +50 -5
- package/docs/CONFIGURATION.md +3 -1
- package/docs/KNOWLEDGE-STORE.md +38 -4
- package/docs/SDK.md +1 -1
- package/package.json +12 -11
- package/scripts/setup.cjs +6 -3
- package/src/cli.ts +60 -7
- package/src/config.ts +35 -15
- package/src/core/pi-version.ts +12 -3
- package/src/healthcheck/index.ts +18 -0
- package/src/index.ts +5 -3
- package/src/infrastructure/browser/thread-worker.mjs +7 -0
- package/src/infrastructure/knowledge-store-service.ts +29 -4
- package/src/knowledge/availability.ts +33 -10
- package/src/knowledge/index.ts +17 -6
- package/src/knowledge/store-operations.ts +177 -0
- package/src/knowledge/store-schema.ts +58 -0
- package/src/knowledge/store.ts +118 -21
- package/src/research-config.ts +58 -18
package/dist/thread-worker.mjs
CHANGED
|
@@ -2699,6 +2699,13 @@ var ConfigSchema = Type.Object({
|
|
|
2699
2699
|
WORKER_CONCURRENCY: Type.Number({ minimum: 1, maximum: 10, default: 2 }),
|
|
2700
2700
|
/** Knowledge store isolation mode (default: 'global' — one shared store across every directory) */
|
|
2701
2701
|
KNOWLEDGE_STORE_MODE: Type.Union([Type.Literal("none"), Type.Literal("project"), Type.Literal("global")], { default: "global" }),
|
|
2702
|
+
/** Knowledge store retrieval strategy (default: 'vector' — current hybrid dense+sparse behavior).
|
|
2703
|
+
* 'vector' = hybrid search (embedding similarity + BM25/FTS fused via RRF; requires the
|
|
2704
|
+
* optional @huggingface/transformers dependency and a downloaded embedding model).
|
|
2705
|
+
* 'bm25' = pure lexical/BM25 retrieval over the same stored findings — NO embedding model:
|
|
2706
|
+
* never initialized, downloaded, or invoked. Uses a separate lexical table in the same
|
|
2707
|
+
* LanceDB database, so existing vector stores are untouched (no migration). */
|
|
2708
|
+
KNOWLEDGE_STORE_RETRIEVAL: Type.Union([Type.Literal("vector"), Type.Literal("bm25")], { default: "vector" }),
|
|
2702
2709
|
/** Embedding model to use for the knowledge store */
|
|
2703
2710
|
EMBEDDING_MODEL: Type.String({ default: "onnx-community/granite-embedding-small-english-r2-ONNX" }),
|
|
2704
2711
|
/** Hardware backend for embeddings: 'auto' (probe WebGPU viability out-of-process, fall back to CPU), 'webgpu' (force), or 'cpu' (force).
|
package/docs/AGENT-SKILL.md
CHANGED
|
@@ -125,7 +125,7 @@ Manual. Symlink the directory into any agent's skills folder yourself:
|
|
|
125
125
|
|
|
126
126
|
### Prerequisites
|
|
127
127
|
|
|
128
|
-
- Node.js >= 22.
|
|
128
|
+
- Node.js >= 22.22.2
|
|
129
129
|
- `pi-research` installed where the launcher can find it, plus a configured model
|
|
130
130
|
(`PI_RESEARCH_MODEL`) + API key. See [Configuration](CONFIGURATION.md).
|
|
131
131
|
|
package/docs/CHANGELOG.md
CHANGED
|
@@ -5,8 +5,39 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
## [1.6.
|
|
8
|
+
## [1.6.20] - 2026-09-21
|
|
9
9
|
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **BM25-only (lexical) knowledge-store retrieval mode: the store can now run entirely without an embedding model** (merged PR #14 by @akmalayari). A new project-scoped setting, `PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL` (`vector` — the default and unchanged hybrid behavior — or `bm25`), selects how stored findings are retrieved. In `bm25` mode, retrieval is pure BM25 over LanceDB's Tantivy full-text indexes (the same lexical half the hybrid path already used, now standing alone over the summary text and full page Markdown) — the `@huggingface/transformers` dependency is never resolved, loaded, downloaded, or invoked, so a bm25-only installation works with `--omit=optional` and no model download. Each strategy keeps its own table in the same database directory (`knowledge` vs `knowledge-bm25`), so switching requires no migration and existing vector stores are untouched. The mode is exposed everywhere the scoping mode is: `knowledge-config set retrieval <vector|bm25>` and `show` on the CLI, a Knowledge Retrieval entry in the `/research-config` menu (embedding model/device entries hide in bm25 mode), the healthcheck, and the availability probe — which in bm25 mode no longer demands the optional embedding package. New tests cover the lexical round-trip, BM25 ranking sanity, scope isolation, cross-mode coexistence, per-mode availability probing, config parsing, and a hard embedder-factory-never-called isolation assertion; docs (KNOWLEDGE-STORE.md, CONFIGURATION.md, `.env.example`) document the three-way scoping × retrieval story.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- **pi packages updated to the 0.87.0 line and the compatibility floor/tested-ceiling moved to match.** `@earendil-works/pi-ai`, `pi-coding-agent`, `pi-server` and `pi-tui` move `>=0.85.0 <1` → `>=0.87.0 <1` (all four verified at 0.87.0); `PI_MIN_VERSION` and `PI_TESTED_MAX_VERSION` move 0.85.0 → 0.87.0 with the floor's rationale updated for the 0.86 pi-ai TranscriptContext / JSON-value / fail-closed-user_bash changes and the 0.87 `SessionEntry` union (`ContextEditEntry`), SessionManager-canonical provider context, and `shouldStopAfterTurn` → `finishTurn` boundary. All four type-checks, lint, and the full unit suite (3,120 tests) pass against the 0.87.0 packages; `src/index.ts`'s runtime-check comment tracks the new floor.
|
|
17
|
+
- **`@lancedb/lancedb` moves 0.38.0 → 0.39.0** (same-semver native store; the full unit suite passes against it, and the `apache-arrow` hold below was re-verified against the new native reader — the 21.2.0 IPC failure reproduces identically under 0.39.0).
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- **CodeQL alert #267 (`js/unused-local-variable`, note): an unused `vi` import left in the PR #14 test file `store-bm25.test.ts` is removed.** Caught by the CodeQL run on the merge commit; no other open alerts remain.
|
|
22
|
+
- **Dependency holds re-verified and enforced again after an attempted refresh:** `apache-arrow` stays 21.1.0 — 21.2.0 breaks `@lancedb/lancedb` 0.39.0's IPC schema parsing (the RangeOutOfBounds footer error reproduces in the test runner; a create-then-read of the store schema fails at `createTable`), confirming the existing hold against the new native reader; `playwright-core` stays 1.60.0 (camoufox-js 0.12.0's peer range still caps `<1.61.0` — 1.63.0 fails `npm install` with ERESOLVE); `@huggingface/transformers` stays 4.2.0 (4.3.0 pulls `onnxruntime-node` 1.30.0 and reshuffles the optional native stack for no needed gain, per the 1.6.18 hold); `typebox` moves 1.3.7 → 1.3.27, realigning the lockstep rule with the new host line (pi 0.87.0's own packages pin 1.3.27 — 1.3.34 was briefly tried and reverted for breaking that rule). Dev-only in-range bumps: `eslint` 10.10.0 → 10.11.0, `@typescript-eslint/*` 8.70.0 → 8.70.1.
|
|
23
|
+
|
|
24
|
+
## [1.6.19] - 2026-09-18
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
|
|
28
|
+
- **`scripts/verify-changelog.cjs`: a release gate that the tagged commit carries a dated changelog section for its version.** The release path validated the tag against `package.json` and `SKILL.md` but never against `docs/CHANGELOG.md`, so a release could be tagged with its changes still sitting under `## [Unreleased]` — which happened twice. `v1.6.12`'s block was later folded into `[1.6.13]` (misattributing it), and `v1.6.17`'s was overwritten by the next release and lost outright. The gate fails closed on a missing, undated, empty, duplicated, or non-newest section; it is unit-tested (`test/unit/scripts/verify-changelog.test.ts`, 11 cases) and runs in CI's lint job and in the release workflow's pre-publish job. The two historical gaps are repaired in the same pass: `[1.6.17]` is restored from git, `[1.6.12]` is reconstructed from the commits in `v1.6.11..v1.6.12` (the Stack Exchange clickable-link fix and the explicit `vite` devDependency, both of which `[1.6.13]` had absorbed, plus the previously undocumented Japanese translation), and `[1.6.18]` gains its missing date.
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- **README: the adm-zip advisory bullet is removed from the requirements/limitations list.** It only asserted the absence of advisories, which is the audit gate's job (`docs/AUDIT-GATE.md`), not a user-facing limitation. The stealth-browser size claim is corrected from "~500MB" to "1GB+ installed" (the Linux asset downloads at ~630MB and unpacks to ~1.3GB).
|
|
33
|
+
- **Supported Node floor raised from 22.19.0 to 22.22.2** (`engines.node`, `.nvmrc`, README, `docs/AGENT-SKILL.md`, `docs/SDK.md`, all three translations, the `agent-skill` README, the `scripts/setup.cjs` check + warning, and the lockfile root). `jsdom` 30 — a production dependency — and `npm` 12, which CI's install jobs run, both declare `^22.22.2 || ^24.15.0 || >=26.0.0`, so the old floor installed a tree outside its own dependency's support. CI's npm upgrade moved to a separate-prefix install as a result (see Fixed).
|
|
34
|
+
- **Two stale in-code comments that still named the pre-0.85.0 pi floor are corrected to 0.85.0** (`src/core/pi-version.ts`, `src/index.ts`). No behavioral or dependency change.
|
|
35
|
+
|
|
36
|
+
### Fixed
|
|
37
|
+
|
|
38
|
+
- **CI no longer upgrades npm by replacing the bundled npm in place.** Node 22.22.2 bundles npm 10.9.7, which cannot install a new npm over itself: the reify deletes npm's own `node_modules` entries while the running process still needs them, and it dies with `Cannot find module 'promise-retry'` (reproduced locally on v22.22.2; CI runs 35414345489 and 35414867837, the ubuntu/macOS unit-test and validate legs). `--force`, `--ignore-scripts` and `--no-audit` all reproduce it, so `--force` was never the fix and the original comment's engines rationale was wrong. `scripts/use-npm12.cjs` installs npm 12 into a separate prefix and puts it on PATH; the release workflow's `publish` job does the same inline for npm 11, which self-upgraded on the same Node and would otherwise have failed the publish.
|
|
39
|
+
|
|
40
|
+
## [1.6.18] - 2026-09-16
|
|
10
41
|
### Fixed
|
|
11
42
|
|
|
12
43
|
- **`buildSafeOptions` no longer sends the literal `'off'` reasoning level into pi-ai's compat API (merged PR #13 by @lukey03).** pi-ai 0.85.1's compat path treats any truthy `reasoning` as thinking-enabled, and `'off'` has no entry in its thinking-budget table, so `maxTokens` was computed as `NaN` and serialized as `max_tokens: null` — every coordinator/plan call 400'd on providers that validate the cap (observed on Fireworks' anthropic-messages endpoint, where research never got past the first planning call). `'off'` now crosses the boundary as `undefined`, pi-ai's no-reasoning value; explicit non-off levels, caller precedence, and the maxTokens cap are unchanged. The new `build-safe-options.test.ts` drives the real compat `completeSimple` against a stubbed transport and asserts on the actual request body (5 of its 9 tests fail pre-patch); the one coordinator-boundary assertion that expected the literal `'off'` was updated to the compat contract.
|
|
@@ -20,6 +51,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
20
51
|
- **README: the "Currently recommended model" line is removed** (leaving the README with no model endorsement), and the leftover double blank line it left behind is fixed.
|
|
21
52
|
- **Dev dependency lockfile refresh, all in-range: `vitest` 5.0.0 → 5.0.1, `@vitest/coverage-v8` 5.0.0 → 5.0.1, `vite` 8.2.2 → 8.3.0, `dependency-cruiser` 18.2.0 → 18.3.1, `jsdom` 30.0.1 → 30.1.0, `webgpu` 0.6.0 → 0.6.1.** Deliberate holds re-verified and unchanged: `typebox` stays 1.3.7 (lockstep with the host pi 0.85.1 line), `apache-arrow` stays 21.1.0 (21.2.0 breaks LanceDB IPC), `playwright-core` stays 1.60.0 (camoufox-js 0.12.0 peers `<1.61.0`), `typescript` stays `<6.1.0` (@typescript-eslint peer ceiling), and `@huggingface/transformers` stays 4.2.0 (4.3.0 pulls an `onnxruntime-web` dev build, `1.31.0-dev.20260914`, and reshuffles the optional native stack for no needed gain). The full gate suite is green: lint, all four type-checks (TS 6 and TS 7 native, src and tests), 3,073 unit tests, dependency-cruiser, build + manifest verify, and the audit gate with an empty allowlist.
|
|
22
53
|
|
|
54
|
+
## [1.6.17] - 2026-09-09
|
|
55
|
+
|
|
56
|
+
### Changed
|
|
57
|
+
|
|
58
|
+
- **README: the adm-zip limitation bullet is trimmed to its essentials.** It no longer walks through the consumer-CI consequence (`npm audit` exiting nonzero until the upstream fix ships) or re-states the verified mechanics — installs succeed, don't run `npm audit fix --force`, details in `docs/AUDIT-GATE.md` — because pi-research is a research tool, not a CI component, and the full verified impact matrix lives in the runbook. No behavioral or dependency change.
|
|
59
|
+
|
|
23
60
|
## [1.6.16] - 2026-09-09
|
|
24
61
|
|
|
25
62
|
### Added
|
|
@@ -64,14 +101,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
64
101
|
|
|
65
102
|
- **`npm ci` under npm 12 rejected the lockfile an `npm update` had rewritten** (this release cycle's own dep refresh): the rewrite dropped the `node-domexception@1.0.28` alias entry backing the `@nolyfill/domexception` override while leaving a stale real `node-domexception@1.0.0` copy nested under `@earendil-works/pi-coding-agent`. npm 11 (the local dev npm) accepted the tree; CI's npm 12 is stricter and failed every `npm ci --legacy-peer-deps` leg. The lockfile was regenerated with npm 12 itself, which collapses the stale nested copy so the override is the only resolution; `npm ci --legacy-peer-deps` now validates clean under both npm 11 and npm 12, and CI is green cross-OS.
|
|
66
103
|
|
|
67
|
-
- **Stack Exchange tool output can no longer open a real browser tab when clicked.** The tool's formatters emitted markdown links (`[title](link)`) and bare URLs, and question/answer bodies carried third-party markup — any markdown-rendering surface (assistant replies, research reports, the terminal UI, which turns markdown links AND bare URLs into OSC 8 terminal hyperlinks and opens them in the system browser on mouse press+release) turned these into click-to-open-tab targets in the middle of scrolling tool output. New `src/utils/plain-url.ts` fixes this at the source, cross-platform and renderer-independent: `plainUrl()` wraps every emitted URL in backticks (code spans tokenize before autolink/link parsing in every CommonMark renderer, so a code-spanned URL can never become a link but stays byte-exact and copyable for the model), and `fenceBlock()` wraps third-party body content in a fenced code block whose length grows past any backtick run in the content (embedded links, headings and tables render inert; content containing its own ``` fences cannot close the fence early). All stackexchange formatters (questions, answers, users, sites, compact variants, and the `get`+compact path) now emit code-spanned URLs and own-line-fenced bodies — fence placement matters: an opening fence inline after `- **Body:**` is NOT recognized as a code block by CommonMark and was verified (against pi's bundled `marked`) to still render embedded links clickable, so the fence starts on its own line. The tool's prompt guidelines now also instruct the model never to re-emit result URLs as markdown links, covering the assistant-echo path. Pinned by `test/unit/stackexchange/output/no-clickable-links.test.ts`, which scans each formatter's clickable surface (post code-span/fence stripping) for link syntax and — when `marked` is resolvable — renders the actual output and asserts zero anchor elements.
|
|
68
|
-
|
|
69
|
-
- **CI: declare `vite` as an explicit devDependency.** CI installs with `--legacy-peer-deps`, which skips peer dependency resolution — `vite` was only reachable as a transitive peer and the install could not resolve it. Now declared directly.
|
|
70
|
-
|
|
71
104
|
### Verified
|
|
72
105
|
|
|
73
106
|
- 3,064 unit tests over 242 files, ESLint clean, all four type-checks (TS 6 src + tests, TS 7 native src + tests), dependency-cruiser and madge clean (190 modules, no circular dependencies), build, and `npm ci --dry-run` manifest verification; `npm audit` reports 0 vulnerabilities. The packed tarball was installed into a clean project: pi 0.85.1 resolves fresh, the documented dual apache-arrow layout (21.1.0 nested / 18.1.0 hoisted for LanceDB) reproduces, and the installed CLI boots. CI green cross-OS (ubuntu ×2, macOS, Windows) including the consumer fresh-install and Arrow/LanceDB interop legs. For this cycle additionally: the tarball installed and the CLI booted under npm 10, npm 11, and npm 12 alike, with the dual apache-arrow layout identical across all three.
|
|
74
107
|
|
|
108
|
+
## [1.6.12] - 2026-09-07
|
|
109
|
+
|
|
110
|
+
### Added
|
|
111
|
+
|
|
112
|
+
- **Consolidated Japanese translation** (`docs/DOCUMENTATION-JA.md`), listed after the zh-CN doc in the README.
|
|
113
|
+
|
|
114
|
+
### Fixed
|
|
115
|
+
|
|
116
|
+
- **Stack Exchange tool output can no longer open a real browser tab when clicked.** The tool's formatters emitted markdown links (`[title](link)`) and bare URLs, and question/answer bodies carried third-party markup — any markdown-rendering surface (assistant replies, research reports, the terminal UI, which turns markdown links AND bare URLs into OSC 8 terminal hyperlinks and opens them in the system browser on mouse press+release) turned these into click-to-open-tab targets in the middle of scrolling tool output. New `src/utils/plain-url.ts` fixes this at the source, cross-platform and renderer-independent: `plainUrl()` wraps every emitted URL in backticks (code spans tokenize before autolink/link parsing in every CommonMark renderer, so a code-spanned URL can never become a link but stays byte-exact and copyable for the model), and `fenceBlock()` wraps third-party body content in a fenced code block whose length grows past any backtick run in the content (embedded links, headings and tables render inert; content containing its own ``` fences cannot close the fence early). All stackexchange formatters (questions, answers, users, sites, compact variants, and the `get`+compact path) now emit code-spanned URLs and own-line-fenced bodies — fence placement matters: an opening fence inline after `- **Body:**` is NOT recognized as a code block by CommonMark and was verified (against pi's bundled `marked`) to still render embedded links clickable, so the fence starts on its own line. The tool's prompt guidelines now also instruct the model never to re-emit result URLs as markdown links, covering the assistant-echo path. Pinned by `test/unit/stackexchange/output/no-clickable-links.test.ts`, which scans each formatter's clickable surface (post code-span/fence stripping) for link syntax and — when `marked` is resolvable — renders the actual output and asserts zero anchor elements.
|
|
117
|
+
|
|
118
|
+
- **CI: declare `vite` as an explicit devDependency.** CI installs with `--legacy-peer-deps`, which skips peer dependency resolution — `vite` was only reachable as a transitive peer and the install could not resolve it. Now declared directly.
|
|
119
|
+
|
|
75
120
|
## [1.6.11] - 2026-09-04
|
|
76
121
|
|
|
77
122
|
### Changed
|
package/docs/CONFIGURATION.md
CHANGED
|
@@ -27,6 +27,7 @@ variables still take effect.) A setting is written to one of two scopes:
|
|
|
27
27
|
|---------|-------|--------|---------|
|
|
28
28
|
| `/research` depth | project | normal · deep · ultra | `PI_RESEARCH_DEFAULT_RESEARCH_DEPTH` |
|
|
29
29
|
| Knowledge Mode | project | none · project · global | `PI_RESEARCH_KNOWLEDGE_STORE_MODE` |
|
|
30
|
+
| Knowledge Retrieval | project | vector · bm25 | `PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL` |
|
|
30
31
|
| Researcher Timeout | user | 3 · 5 · 10 · 15 · 20 · 30 (minutes) | `PI_RESEARCH_TIMEOUT_MS` |
|
|
31
32
|
| Max Concurrency | user | 1 – 5 | `PI_RESEARCH_MAX_RESEARCHERS` |
|
|
32
33
|
| Scrape Batches | user | unlimited · 1 · 2 · 3 · 5 · 10 · 15 | `PI_RESEARCH_MAX_SCRAPE_BATCHES` |
|
|
@@ -127,7 +128,8 @@ See the [knowledge store doc](KNOWLEDGE-STORE.md) for what each value does.
|
|
|
127
128
|
|
|
128
129
|
| Variable | Default | Range | Description |
|
|
129
130
|
|----------|---------|-------|-------------|
|
|
130
|
-
| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). Independently of this setting, the store is cleanly OFF when its required packages are not installed (optional `@huggingface/transformers` skipped at install, broken `@lancedb/lancedb`): init fails fast instead of retry-storming, and `pi-research knowledge-config`, the `/research-config` menu, and the healthcheck all name the missing package and the repair. |
|
|
131
|
+
| `PI_RESEARCH_KNOWLEDGE_STORE_MODE` (TUI) `[project]` | `global` | none · project · global | Store scope: one shared store across every directory (`global`), scoped to the current directory (`project`), or disabled (`none`). Independently of this setting, the store is cleanly OFF when its required packages are not installed (optional `@huggingface/transformers` skipped at install, broken `@lancedb/lancedb`): init fails fast instead of retry-storming, and `pi-research knowledge-config`, the `/research-config` menu, and the healthcheck all name the missing package and the repair. In `bm25` retrieval mode the embedding package is not required at all. |
|
|
132
|
+
| `PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL` (TUI) `[project]` | `vector` | vector · bm25 | Retrieval strategy: `vector` = hybrid (embedding similarity + BM25 fused via RRF; needs the optional embedding model), `bm25` = pure lexical BM25 (no embedding model — never downloaded or initialized; works with `--omit=optional`). Each strategy keeps its own table in the same database directory, so switching requires no migration. |
|
|
131
133
|
| `PI_RESEARCH_EMBEDDING_MODEL` (TUI) | `onnx-community/granite-embedding-small-english-r2-ONNX` | — | Embedding model. Changing it clears the store and starts fresh. |
|
|
132
134
|
| `PI_RESEARCH_EMBEDDING_DEVICE` (TUI) | `auto` | auto · webgpu · cpu | Inference backend. `auto` probes WebGPU viability out-of-process and falls back to CPU; `cpu` forces CPU; `webgpu` forces the GPU path with no probe (advanced — can hard-crash on a software GPU). The TUI exposes only `auto` (as "GPU") and `cpu`. |
|
|
133
135
|
| `PI_RESEARCH_CACHE_TTL_DAYS` (TUI) | `30` | 1–365 | How long cached findings are kept before eviction. |
|
package/docs/KNOWLEDGE-STORE.md
CHANGED
|
@@ -58,6 +58,35 @@ The embedding model is lazy — it only downloads and initializes the first time
|
|
|
58
58
|
the store is actually written or searched, so the `global` default adds no startup
|
|
59
59
|
cost until a run caches its first page.
|
|
60
60
|
|
|
61
|
+
### Retrieval strategies: vector (hybrid) and bm25 (lexical)
|
|
62
|
+
|
|
63
|
+
Independent of scope, the store's retrieval strategy is set by
|
|
64
|
+
`PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL` (a project-scoped setting, default
|
|
65
|
+
`vector`):
|
|
66
|
+
|
|
67
|
+
| Strategy | Behavior |
|
|
68
|
+
|----------|----------|
|
|
69
|
+
| `vector` (default) | Hybrid search: embedding similarity + BM25 keyword matching over the stored summaries and full page text, fused via reciprocal rank fusion (RRF). Richest matching, especially for paraphrases. Requires the optional `@huggingface/transformers` dependency and a downloaded embedding model. |
|
|
70
|
+
| `bm25` | Pure lexical search: BM25 ranking over the same stored summaries and full page text. **No embedding model** — never downloaded, initialized, or invoked — so a bm25-only installation works with `--omit=optional` and adds no model download. |
|
|
71
|
+
|
|
72
|
+
Change the strategy for the current directory with the `/research-config` TUI
|
|
73
|
+
(Knowledge Retrieval), or with `pi-research knowledge-config set retrieval
|
|
74
|
+
<vector|bm25>` on the standalone CLI. Like Knowledge Mode, it persists to the
|
|
75
|
+
per-directory project registry and applies without a restart.
|
|
76
|
+
|
|
77
|
+
The two strategies keep **separate tables** in the same database directory
|
|
78
|
+
(`knowledge` for vector, `knowledge-bm25` for bm25). An existing vector store is
|
|
79
|
+
never touched by bm25 mode and vice versa — switching strategies requires no
|
|
80
|
+
migration, and each mode keeps its own history while the other is active. In
|
|
81
|
+
`bm25` mode, `exportKnowledge()` omits vectors, and the embedding settings
|
|
82
|
+
(`PI_RESEARCH_EMBEDDING_MODEL`, `PI_RESEARCH_EMBEDDING_DEVICE`,
|
|
83
|
+
`PI_RESEARCH_EMBEDDING_MODEL_INIT_TIMEOUT_MS`) are ignored entirely.
|
|
84
|
+
|
|
85
|
+
BM25 ranking uses Tantivy (LanceDB's full-text engine) with its default Unicode
|
|
86
|
+
tokenization and lowercasing, over both the summary text and the full page
|
|
87
|
+
Markdown — so lexical mode matches the same fields the hybrid mode's keyword
|
|
88
|
+
half does.
|
|
89
|
+
|
|
61
90
|
### How a run uses the store
|
|
62
91
|
|
|
63
92
|
The store is driven by the orchestrator, not called ad hoc by researcher agents,
|
|
@@ -76,6 +105,9 @@ synthesized answer with citations — or reports that live research is needed.
|
|
|
76
105
|
|
|
77
106
|
### Embeddings and the model
|
|
78
107
|
|
|
108
|
+
Applies to the `vector` (hybrid) retrieval strategy only — `bm25` mode runs with
|
|
109
|
+
no embedding model at all.
|
|
110
|
+
|
|
79
111
|
Embeddings are computed locally with
|
|
80
112
|
[`@huggingface/transformers`](https://github.com/huggingface/transformers.js) over
|
|
81
113
|
ONNX. The default model is
|
|
@@ -173,7 +205,8 @@ keep history longer.
|
|
|
173
205
|
|
|
174
206
|
### Changing the model: migration
|
|
175
207
|
|
|
176
|
-
|
|
208
|
+
Applies to the `vector` retrieval strategy. When the configured embedding model
|
|
209
|
+
differs from the one the stored vectors were
|
|
177
210
|
built with, the store is migrated according to `PI_RESEARCH_MIGRATION_STRATEGY`:
|
|
178
211
|
|
|
179
212
|
| Strategy | What happens |
|
|
@@ -209,12 +242,13 @@ indices are pruned to keep it bounded. There is no manual maintenance command.
|
|
|
209
242
|
| Setting | Variable | Default |
|
|
210
243
|
|---------|----------|---------|
|
|
211
244
|
| Knowledge Mode (project-scoped) | `PI_RESEARCH_KNOWLEDGE_STORE_MODE` | `global` |
|
|
212
|
-
|
|
|
213
|
-
| Embedding
|
|
245
|
+
| Retrieval strategy (project-scoped) | `PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL` | `vector` |
|
|
246
|
+
| Embedding model (vector mode only) | `PI_RESEARCH_EMBEDDING_MODEL` | `onnx-community/granite-embedding-small-english-r2-ONNX` |
|
|
247
|
+
| Embedding device (vector mode only) | `PI_RESEARCH_EMBEDDING_DEVICE` | `auto` |
|
|
214
248
|
| Cache retention (days) | `PI_RESEARCH_CACHE_TTL_DAYS` | `30` |
|
|
215
249
|
| Migration strategy | `PI_RESEARCH_MIGRATION_STRATEGY` | `backup` |
|
|
216
250
|
| Database directory | `PI_RESEARCH_KNOWLEDGE_DIR` | `~/.pi/research/knowledge_db` |
|
|
217
|
-
| Model init timeout (ms) | `PI_RESEARCH_EMBEDDING_MODEL_INIT_TIMEOUT_MS` | `300000` |
|
|
251
|
+
| Model init timeout, vector mode only (ms) | `PI_RESEARCH_EMBEDDING_MODEL_INIT_TIMEOUT_MS` | `300000` |
|
|
218
252
|
| Re-probe WebGPU | `PI_RESEARCH_WEBGPU_REPROBE` | _(unset)_ |
|
|
219
253
|
|
|
220
254
|
See [CONFIGURATION.md](CONFIGURATION.md) for the full configuration model, and
|
package/docs/SDK.md
CHANGED
|
@@ -50,7 +50,7 @@ global file entirely and run purely from defaults + `process.env` + `options.con
|
|
|
50
50
|
> `ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING`. That applies to
|
|
51
51
|
> `--experimental-transform-types` exactly as it does to
|
|
52
52
|
> `--experimental-strip-types`, so neither flag helps here; a loader (or the pi
|
|
53
|
-
> host) is mandatory. (`engines.node` is `>=22.
|
|
53
|
+
> host) is mandatory. (`engines.node` is `>=22.22.2`.)
|
|
54
54
|
|
|
55
55
|
```typescript
|
|
56
56
|
import {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lincoln504/pi-research",
|
|
3
|
-
"version": "1.6.
|
|
3
|
+
"version": "1.6.20",
|
|
4
4
|
"description": "Web research for your agents with smart and safe tooling + knowledge store",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
"access": "public"
|
|
74
74
|
},
|
|
75
75
|
"engines": {
|
|
76
|
-
"node": ">=22.
|
|
76
|
+
"node": ">=22.22.2"
|
|
77
77
|
},
|
|
78
78
|
"scripts": {
|
|
79
79
|
"lint": "npx eslint --config config/tooling/eslint.config.js src/",
|
|
@@ -104,6 +104,7 @@
|
|
|
104
104
|
"install:system-deps": "node scripts/setup.cjs --system-deps",
|
|
105
105
|
"deps:check": "depcruise --config config/tooling/dependency-cruiser.cjs --output-type err-long src",
|
|
106
106
|
"deps:generate": "npx --yes madge@8.0.0 --image docs/deps.svg src/index.ts",
|
|
107
|
+
"verify:changelog": "node scripts/verify-changelog.cjs",
|
|
107
108
|
"analyze:dependencies": "ncu",
|
|
108
109
|
"analyze:compatibility": "depngn"
|
|
109
110
|
},
|
|
@@ -114,12 +115,12 @@
|
|
|
114
115
|
"image": "https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/hero.png"
|
|
115
116
|
},
|
|
116
117
|
"dependencies": {
|
|
117
|
-
"@earendil-works/pi-ai": ">=0.
|
|
118
|
-
"@earendil-works/pi-coding-agent": ">=0.
|
|
119
|
-
"@earendil-works/pi-server": ">=0.
|
|
120
|
-
"@earendil-works/pi-tui": ">=0.
|
|
118
|
+
"@earendil-works/pi-ai": ">=0.87.0 <1",
|
|
119
|
+
"@earendil-works/pi-coding-agent": ">=0.87.0 <1",
|
|
120
|
+
"@earendil-works/pi-server": ">=0.87.0 <1",
|
|
121
|
+
"@earendil-works/pi-tui": ">=0.87.0 <1",
|
|
121
122
|
"@kreuzberg/html-to-markdown-node": "^3.7.2",
|
|
122
|
-
"@lancedb/lancedb": "0.
|
|
123
|
+
"@lancedb/lancedb": "0.39.0",
|
|
123
124
|
"apache-arrow": "21.1.0",
|
|
124
125
|
"bgutils-js": "^4.0.3",
|
|
125
126
|
"camoufox-js": "^0.12.0",
|
|
@@ -130,7 +131,7 @@
|
|
|
130
131
|
"pdf-oxide-wasm": "^0.3.77",
|
|
131
132
|
"playwright-core": "1.60.0",
|
|
132
133
|
"poolifier": "^5.3.2",
|
|
133
|
-
"typebox": "1.3.
|
|
134
|
+
"typebox": "1.3.27",
|
|
134
135
|
"undici": "^8.10.2",
|
|
135
136
|
"youtubei.js": "^18.0.0"
|
|
136
137
|
},
|
|
@@ -156,12 +157,12 @@
|
|
|
156
157
|
"devDependencies": {
|
|
157
158
|
"@eslint/js": "^10.0.1",
|
|
158
159
|
"@types/jsdom": "^30.0.0",
|
|
159
|
-
"@typescript-eslint/eslint-plugin": "
|
|
160
|
-
"@typescript-eslint/parser": "
|
|
160
|
+
"@typescript-eslint/eslint-plugin": "8.70.1",
|
|
161
|
+
"@typescript-eslint/parser": "8.70.1",
|
|
161
162
|
"@vitest/coverage-v8": "^5.0.1",
|
|
162
163
|
"dependency-cruiser": "^18.2.0",
|
|
163
164
|
"depngn": "^0.6.0",
|
|
164
|
-
"eslint": "
|
|
165
|
+
"eslint": "10.11.0",
|
|
165
166
|
"npm-check-updates": "^23.1.0",
|
|
166
167
|
"typescript": ">=6.0.3 <6.1.0",
|
|
167
168
|
"vite": "^8.2.2",
|
package/scripts/setup.cjs
CHANGED
|
@@ -23,9 +23,12 @@ const isLinux = process.platform === 'linux';
|
|
|
23
23
|
const isDarwin = process.platform === 'darwin';
|
|
24
24
|
const isWindows = process.platform === 'win32';
|
|
25
25
|
|
|
26
|
-
const [nodeMajor, nodeMinor] = process.version.replace('v', '').split('.').map((n) => parseInt(n, 10));
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
const [nodeMajor, nodeMinor, nodePatch] = process.version.replace('v', '').split('.').map((n) => parseInt(n, 10));
|
|
27
|
+
// Minimum is 22.22.2 — the floor `jsdom` 30 (a runtime dependency) and npm 12 both declare.
|
|
28
|
+
const belowMinimum =
|
|
29
|
+
nodeMajor < 22 || (nodeMajor === 22 && (nodeMinor < 22 || (nodeMinor === 22 && nodePatch < 2)));
|
|
30
|
+
if (belowMinimum) {
|
|
31
|
+
console.warn(`WARNING: Node.js ${process.version} is below the minimum (>=22.22.2). Upgrade to 22.22.2+.`);
|
|
29
32
|
}
|
|
30
33
|
|
|
31
34
|
/**
|
package/src/cli.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* pi-research research "<query>" [--depth N] [--model provider/id]
|
|
9
9
|
* pi-research knowledge "<q>" ["<q2>" ...]
|
|
10
10
|
* pi-research knowledge-config [set <none|project|global>]
|
|
11
|
+
* pi-research knowledge-config set retrieval <vector|bm25>
|
|
11
12
|
* pi-research status [--json]
|
|
12
13
|
* pi-research help
|
|
13
14
|
*
|
|
@@ -895,6 +896,32 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
|
|
|
895
896
|
// silently breaking the documented `--json: Emit a JSON object` contract for
|
|
896
897
|
// this one subcommand.
|
|
897
898
|
try {
|
|
899
|
+
if (kc.action === 'set-retrieval') {
|
|
900
|
+
const before = getConfig(cwd, 'cli').KNOWLEDGE_STORE_RETRIEVAL;
|
|
901
|
+
const cfg = { ...getConfig(cwd, 'cli'), KNOWLEDGE_STORE_RETRIEVAL: kc.retrieval! };
|
|
902
|
+
// Persist ONLY the retrieval strategy for this directory — mirrors the
|
|
903
|
+
// scoping-mode `set` branch above (changedKeys-scoped registry write).
|
|
904
|
+
saveConfig(cfg, 'local', cwd, ['PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL']);
|
|
905
|
+
resetConfig();
|
|
906
|
+
const info = describeKnowledgeStoreMode(cwd, 'cli');
|
|
907
|
+
const overridden = info.retrieval !== kc.retrieval;
|
|
908
|
+
if (kc.json) {
|
|
909
|
+
toStdout(pretty({ command: 'knowledge-config', action: 'set-retrieval', requested: kc.retrieval, previous: before, saved: true, effectiveOverriddenBy: overridden ? info.retrievalOrigin : null, ...info, cwd }));
|
|
910
|
+
return EXIT.OK;
|
|
911
|
+
}
|
|
912
|
+
toStdout(
|
|
913
|
+
`knowledge store retrieval for this directory: ${before} -> ${kc.retrieval} (saved per-directory)\n` +
|
|
914
|
+
` strategy: ${kc.retrieval === 'bm25' ? 'lexical BM25 — no embedding model needed' : 'hybrid (embedding + BM25, fused) — needs the optional embedding dependency'}\n` +
|
|
915
|
+
` saved: per-directory (${cwd})\n` +
|
|
916
|
+
` db dir: ${info.dbDir}\n` +
|
|
917
|
+
(info.available ? '' : ` state: ${info.unavailableReason}\n`) +
|
|
918
|
+
(overridden
|
|
919
|
+
? `\nNOTE: the effective retrieval here is still '${info.retrieval}' because a higher-precedence ${info.retrievalOrigin} overrides it. Clear that source for this setting to take effect.\n`
|
|
920
|
+
: ''),
|
|
921
|
+
);
|
|
922
|
+
return EXIT.OK;
|
|
923
|
+
}
|
|
924
|
+
|
|
898
925
|
if (kc.action === 'set') {
|
|
899
926
|
const before = getConfig(cwd, 'cli').KNOWLEDGE_STORE_MODE;
|
|
900
927
|
const cfg = { ...getConfig(cwd, 'cli'), KNOWLEDGE_STORE_MODE: kc.mode! };
|
|
@@ -932,11 +959,14 @@ export async function cmdKnowledgeConfig(kc: NonNullable<ParsedArgs['knowledgeCo
|
|
|
932
959
|
}
|
|
933
960
|
toStdout(
|
|
934
961
|
`knowledge store mode: ${info.mode} (source: ${info.origin})\n` +
|
|
935
|
-
`
|
|
936
|
-
`
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
962
|
+
`knowledge store retrieval: ${info.retrieval} (source: ${info.retrievalOrigin})\n` +
|
|
963
|
+
` scope: ${info.mode === 'none' ? 'disabled here' : info.mode === 'project' ? 'this directory only' : 'shared across all directories'}\n` +
|
|
964
|
+
` strategy: ${info.retrieval === 'bm25' ? 'lexical BM25 — no embedding model needed' : 'hybrid (embedding + BM25, fused)'}\n` +
|
|
965
|
+
` db dir: ${info.dbDir}\n` +
|
|
966
|
+
(info.available ? '' : ` state: ${info.unavailableReason}\n`) +
|
|
967
|
+
`\nchange scope for THIS directory: ${BINARY_NAME} knowledge-config set <none|project|global>\n` +
|
|
968
|
+
`change retrieval for THIS directory: ${BINARY_NAME} knowledge-config set retrieval <vector|bm25>\n` +
|
|
969
|
+
`change the machine-wide defaults: set PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> and/or PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL=<vector|bm25> in ${resolvedConfigPaths().configEnv}\n`,
|
|
940
970
|
);
|
|
941
971
|
return EXIT.OK;
|
|
942
972
|
} catch (err) {
|
|
@@ -1235,7 +1265,7 @@ interface ParsedArgs {
|
|
|
1235
1265
|
command?: string;
|
|
1236
1266
|
research?: ResearchArgs;
|
|
1237
1267
|
knowledge?: { queries: string[]; json?: boolean };
|
|
1238
|
-
knowledgeConfig?: { action: 'show' | 'set'; mode?: 'none' | 'project' | 'global'; json?: boolean };
|
|
1268
|
+
knowledgeConfig?: { action: 'show' | 'set' | 'set-retrieval'; mode?: 'none' | 'project' | 'global'; retrieval?: 'vector' | 'bm25'; json?: boolean };
|
|
1239
1269
|
status?: { json?: boolean };
|
|
1240
1270
|
skill?: { action: 'install' | 'uninstall' | 'status'; json?: boolean; dryRun?: boolean; copy?: boolean };
|
|
1241
1271
|
configPath?: string;
|
|
@@ -1270,6 +1300,10 @@ COMMANDS
|
|
|
1270
1300
|
--json Emit a JSON object.
|
|
1271
1301
|
knowledge-config set <mode> Set the mode for THIS directory (none | project | global);
|
|
1272
1302
|
persisted per-directory. Takes effect on the next run.
|
|
1303
|
+
knowledge-config set retrieval <vector|bm25>
|
|
1304
|
+
Set the retrieval strategy for THIS directory: 'vector'
|
|
1305
|
+
(hybrid embedding+BM25, default) or 'bm25' (pure lexical,
|
|
1306
|
+
no embedding model). Persisted per-directory.
|
|
1273
1307
|
|
|
1274
1308
|
status Show detected config, model/key, and readiness.
|
|
1275
1309
|
--config <path> Layer this config file on top (highest file precedence; base
|
|
@@ -1303,6 +1337,7 @@ CONFIGURE
|
|
|
1303
1337
|
|
|
1304
1338
|
Knowledge store — ON by default in every directory (mode 'global', one shared store).
|
|
1305
1339
|
per-directory: ${BINARY_NAME} knowledge-config set <none|project|global>
|
|
1340
|
+
${BINARY_NAME} knowledge-config set retrieval <vector|bm25>
|
|
1306
1341
|
machine-wide: PI_RESEARCH_KNOWLEDGE_STORE_MODE=<mode> in ${p.configEnv}
|
|
1307
1342
|
'project' scopes the store to the current directory; 'none' disables it here.
|
|
1308
1343
|
|
|
@@ -1421,6 +1456,24 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
1421
1456
|
return out;
|
|
1422
1457
|
}
|
|
1423
1458
|
if (action === 'set') {
|
|
1459
|
+
// Extended form: `knowledge-config set retrieval <vector|bm25>` — switches the
|
|
1460
|
+
// retrieval STRATEGY (the scoping mode uses the original single-arg form).
|
|
1461
|
+
if (positional[1] === 'retrieval') {
|
|
1462
|
+
const retrieval = positional[2];
|
|
1463
|
+
if (retrieval === undefined) {
|
|
1464
|
+
throw new UsageError('knowledge-config set retrieval requires a strategy: vector | bm25.');
|
|
1465
|
+
}
|
|
1466
|
+
if (retrieval !== 'vector' && retrieval !== 'bm25') {
|
|
1467
|
+
throw new UsageError(`invalid retrieval strategy "${retrieval}". Use one of: vector | bm25.`);
|
|
1468
|
+
}
|
|
1469
|
+
if (positional.length > 3) {
|
|
1470
|
+
throw new UsageError(`unexpected argument "${positional[3]}" after "knowledge-config set retrieval ${retrieval}".`);
|
|
1471
|
+
}
|
|
1472
|
+
out.command = 'knowledge-config';
|
|
1473
|
+
out.knowledgeConfig = { action: 'set-retrieval', retrieval, json };
|
|
1474
|
+
out.configPath = configPath;
|
|
1475
|
+
return out;
|
|
1476
|
+
}
|
|
1424
1477
|
const mode = positional[1];
|
|
1425
1478
|
if (mode === undefined) {
|
|
1426
1479
|
throw new UsageError('knowledge-config set requires a mode: none | project | global.');
|
|
@@ -1436,7 +1489,7 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
1436
1489
|
out.configPath = configPath;
|
|
1437
1490
|
return out;
|
|
1438
1491
|
}
|
|
1439
|
-
throw new UsageError(`unknown knowledge-config action "${action}". Use: show | set <none|project|global>.`);
|
|
1492
|
+
throw new UsageError(`unknown knowledge-config action "${action}". Use: show | set <none|project|global> | set retrieval <vector|bm25>.`);
|
|
1440
1493
|
}
|
|
1441
1494
|
|
|
1442
1495
|
if (cmd === 'skill') {
|
package/src/config.ts
CHANGED
|
@@ -50,6 +50,13 @@ export const ConfigSchema = Type.Object({
|
|
|
50
50
|
WORKER_CONCURRENCY: Type.Number({ minimum: 1, maximum: 10, default: 2 }),
|
|
51
51
|
/** Knowledge store isolation mode (default: 'global' — one shared store across every directory) */
|
|
52
52
|
KNOWLEDGE_STORE_MODE: Type.Union([Type.Literal('none'), Type.Literal('project'), Type.Literal('global')], { default: 'global' }),
|
|
53
|
+
/** Knowledge store retrieval strategy (default: 'vector' — current hybrid dense+sparse behavior).
|
|
54
|
+
* 'vector' = hybrid search (embedding similarity + BM25/FTS fused via RRF; requires the
|
|
55
|
+
* optional @huggingface/transformers dependency and a downloaded embedding model).
|
|
56
|
+
* 'bm25' = pure lexical/BM25 retrieval over the same stored findings — NO embedding model:
|
|
57
|
+
* never initialized, downloaded, or invoked. Uses a separate lexical table in the same
|
|
58
|
+
* LanceDB database, so existing vector stores are untouched (no migration). */
|
|
59
|
+
KNOWLEDGE_STORE_RETRIEVAL: Type.Union([Type.Literal('vector'), Type.Literal('bm25')], { default: 'vector' }),
|
|
53
60
|
/** Embedding model to use for the knowledge store */
|
|
54
61
|
EMBEDDING_MODEL: Type.String({ default: 'onnx-community/granite-embedding-small-english-r2-ONNX' }),
|
|
55
62
|
/** Hardware backend for embeddings: 'auto' (probe WebGPU viability out-of-process, fall back to CPU), 'webgpu' (force), or 'cpu' (force).
|
|
@@ -211,6 +218,7 @@ export const DEFAULTS: Config = Value.Create(ConfigSchema);
|
|
|
211
218
|
const LOCAL_SCOPE_KEYS = new Set([
|
|
212
219
|
'PI_RESEARCH_DEFAULT_RESEARCH_DEPTH',
|
|
213
220
|
'PI_RESEARCH_KNOWLEDGE_STORE_MODE',
|
|
221
|
+
'PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL',
|
|
214
222
|
]);
|
|
215
223
|
|
|
216
224
|
/**
|
|
@@ -600,6 +608,10 @@ export interface KnowledgeStoreModeInfo {
|
|
|
600
608
|
mode: 'none' | 'project' | 'global';
|
|
601
609
|
/** Human-readable source of the EFFECTIVE value (highest-precedence source that set it). */
|
|
602
610
|
origin: string;
|
|
611
|
+
/** Knowledge store retrieval strategy ('vector' hybrid — default, or 'bm25' lexical-only). */
|
|
612
|
+
retrieval: 'vector' | 'bm25';
|
|
613
|
+
/** Human-readable source of the EFFECTIVE retrieval value (same precedence walk as `origin`). */
|
|
614
|
+
retrievalOrigin: string;
|
|
603
615
|
/** Physical LanceDB directory backing the store (shared across project/global modes). */
|
|
604
616
|
dbDir: string;
|
|
605
617
|
/**
|
|
@@ -627,34 +639,40 @@ export function describeKnowledgeStoreMode(
|
|
|
627
639
|
iface?: ConfigInterface,
|
|
628
640
|
): KnowledgeStoreModeInfo {
|
|
629
641
|
const cfg = getConfig(cwd, iface);
|
|
630
|
-
const
|
|
631
|
-
const fileHasKey = (p: string): boolean => {
|
|
642
|
+
const fileHasKey = (key: string, p: string): boolean => {
|
|
632
643
|
try {
|
|
633
|
-
return fs.existsSync(p) && parseDotEnv(fs.readFileSync(p, 'utf-8'))[
|
|
644
|
+
return fs.existsSync(p) && parseDotEnv(fs.readFileSync(p, 'utf-8'))[key] !== undefined;
|
|
634
645
|
} catch {
|
|
635
646
|
return false;
|
|
636
647
|
}
|
|
637
648
|
};
|
|
638
649
|
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
650
|
+
/**
|
|
651
|
+
* Walk the same precedence createConfig applies, highest first, for one env key:
|
|
652
|
+
* env var > per-directory project registry > {iface}.env overlay > config.env > default.
|
|
653
|
+
*/
|
|
654
|
+
const originFor = (KEY: string): string => {
|
|
655
|
+
if (process.env[KEY] !== undefined) {
|
|
656
|
+
return 'environment variable';
|
|
657
|
+
}
|
|
643
658
|
let registryHasKey = false;
|
|
644
659
|
try {
|
|
645
660
|
const registry = loadProjectSettingsRegistry();
|
|
646
661
|
registryHasKey = findRegistryEntry(registry, normalizeWorkspacePath(cwd))?.[KEY] !== undefined;
|
|
647
662
|
} catch { /* unreadable registry → not the origin */ }
|
|
648
|
-
if (registryHasKey)
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
663
|
+
if (registryHasKey) return 'this directory (project settings)';
|
|
664
|
+
if (iface && fileHasKey(KEY, getInterfaceEnvFilePath(iface))) return `${iface}.env overlay`;
|
|
665
|
+
if (fileHasKey(KEY, getGlobalEnvFilePath())) return 'config.env (machine-wide default)';
|
|
666
|
+
return 'built-in default';
|
|
667
|
+
};
|
|
652
668
|
|
|
653
669
|
return {
|
|
654
670
|
mode: cfg.KNOWLEDGE_STORE_MODE,
|
|
655
|
-
origin,
|
|
671
|
+
origin: originFor('PI_RESEARCH_KNOWLEDGE_STORE_MODE'),
|
|
672
|
+
retrieval: cfg.KNOWLEDGE_STORE_RETRIEVAL,
|
|
673
|
+
retrievalOrigin: originFor('PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL'),
|
|
656
674
|
dbDir: getDbDir(cfg, cwd),
|
|
657
|
-
...availabilityFields(),
|
|
675
|
+
...availabilityFields(cfg),
|
|
658
676
|
};
|
|
659
677
|
}
|
|
660
678
|
|
|
@@ -665,8 +683,8 @@ export function describeKnowledgeStoreMode(
|
|
|
665
683
|
* throws: the probe catches resolution errors internally and reports them as
|
|
666
684
|
* `available: false`.
|
|
667
685
|
*/
|
|
668
|
-
function availabilityFields(): { available: boolean; unavailableReason: string } {
|
|
669
|
-
const availability = probeKnowledgeStoreAvailability();
|
|
686
|
+
function availabilityFields(cfg?: Config): { available: boolean; unavailableReason: string } {
|
|
687
|
+
const availability = probeKnowledgeStoreAvailability(cfg?.KNOWLEDGE_STORE_RETRIEVAL ?? getConfig().KNOWLEDGE_STORE_RETRIEVAL);
|
|
670
688
|
return {
|
|
671
689
|
available: availability.available,
|
|
672
690
|
unavailableReason: describeKnowledgeStoreUnavailability(availability),
|
|
@@ -924,6 +942,7 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
|
|
|
924
942
|
PI_RESEARCH_WORKER_THREADS: String(config.WORKER_THREADS),
|
|
925
943
|
PI_RESEARCH_WORKER_CONCURRENCY: String(config.WORKER_CONCURRENCY),
|
|
926
944
|
PI_RESEARCH_KNOWLEDGE_STORE_MODE: config.KNOWLEDGE_STORE_MODE,
|
|
945
|
+
PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL: config.KNOWLEDGE_STORE_RETRIEVAL,
|
|
927
946
|
PI_RESEARCH_EMBEDDING_MODEL: config.EMBEDDING_MODEL,
|
|
928
947
|
PI_RESEARCH_EMBEDDING_DEVICE: config.EMBEDDING_DEVICE,
|
|
929
948
|
PI_RESEARCH_SCRAPE_TIMEOUT_MS: String(config.SCRAPE_TIMEOUT_MS),
|
|
@@ -1147,6 +1166,7 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
|
|
|
1147
1166
|
WORKER_THREADS: parseEnvNumber(e, 'PI_RESEARCH_WORKER_THREADS', DEFAULTS.WORKER_THREADS, 1, 10, true),
|
|
1148
1167
|
WORKER_CONCURRENCY: parseEnvNumber(e, 'PI_RESEARCH_WORKER_CONCURRENCY', DEFAULTS.WORKER_CONCURRENCY, 1, 10, true),
|
|
1149
1168
|
KNOWLEDGE_STORE_MODE: parseEnvEnum(e, 'PI_RESEARCH_KNOWLEDGE_STORE_MODE', ['none', 'project', 'global'] as const, 'global'),
|
|
1169
|
+
KNOWLEDGE_STORE_RETRIEVAL: parseEnvEnum(e, 'PI_RESEARCH_KNOWLEDGE_STORE_RETRIEVAL', ['vector', 'bm25'] as const, 'vector'),
|
|
1150
1170
|
EMBEDDING_MODEL: parseEnvString(e, 'PI_RESEARCH_EMBEDDING_MODEL', DEFAULTS.EMBEDDING_MODEL)!,
|
|
1151
1171
|
EMBEDDING_DEVICE: parseEnvEnum(e, 'PI_RESEARCH_EMBEDDING_DEVICE', ['auto', 'webgpu', 'cpu'] as const, DEFAULTS.EMBEDDING_DEVICE),
|
|
1152
1172
|
SCRAPE_TIMEOUT_MS: parseEnvNumber(e, 'PI_RESEARCH_SCRAPE_TIMEOUT_MS', DEFAULTS.SCRAPE_TIMEOUT_MS, 5000, 120000),
|
package/src/core/pi-version.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* - As a pi EXTENSION, the host supplies `@earendil-works/*`. Our package.json
|
|
9
9
|
* range never constrains it, so an in-process check is the only enforcement.
|
|
10
|
-
* - As a standalone CLI/SDK, the range is `>=0.
|
|
10
|
+
* - As a standalone CLI/SDK, the range is `>=0.87.0 <1` and published tarballs
|
|
11
11
|
* carry no lockfile, so every fresh install resolves the newest 0.x at that
|
|
12
12
|
* instant. pi is pre-1.0: a minor bump may break anything under semver, and
|
|
13
13
|
* already has (0.83.0 extended the ResourceLoader contract).
|
|
@@ -74,8 +74,17 @@ export function compareVersions(a: SemverParts, b: SemverParts): number {
|
|
|
74
74
|
* indefinite-hang-in-network-restricted-environments bug that explicit
|
|
75
75
|
* option originally existed to prevent. The floor moved to close that gap,
|
|
76
76
|
* not just because the type changed.
|
|
77
|
+
*
|
|
78
|
+
* 0.86.0 changed the pi-ai provider stream input from Context to normalized
|
|
79
|
+
* TranscriptContext values, restricted ToolCall.arguments / ToolResultMessage
|
|
80
|
+
* details to JSON-compatible values, and made user_bash fail closed; 0.87.0
|
|
81
|
+
* added ContextEditEntry to the SessionEntry union, made SessionManager the
|
|
82
|
+
* canonical provider-context source, and replaced shouldStopAfterTurn with the
|
|
83
|
+
* finishTurn boundary. The SDK paths (session service, model-registry factory)
|
|
84
|
+
* are written against that surface; an older host may resolve but is not a
|
|
85
|
+
* tested or supported target.
|
|
77
86
|
*/
|
|
78
|
-
export const PI_MIN_VERSION: SemverParts = { major: 0, minor:
|
|
87
|
+
export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 87, patch: 0 };
|
|
79
88
|
|
|
80
89
|
/**
|
|
81
90
|
* Newest host line this release was actually exercised against (CI + a real run).
|
|
@@ -83,7 +92,7 @@ export const PI_MIN_VERSION: SemverParts = { major: 0, minor: 85, patch: 0 };
|
|
|
83
92
|
* Compared on MAJOR.MINOR only: a patch bump within a tested line is not a new
|
|
84
93
|
* surface, and warning on it would be noise.
|
|
85
94
|
*/
|
|
86
|
-
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor:
|
|
95
|
+
export const PI_TESTED_MAX_VERSION: SemverParts = { major: 0, minor: 87, patch: 0 };
|
|
87
96
|
|
|
88
97
|
export type PiCompatibilityLevel = 'ok' | 'unparseable' | 'too-old' | 'untested';
|
|
89
98
|
|
package/src/healthcheck/index.ts
CHANGED
|
@@ -196,8 +196,25 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
|
|
|
196
196
|
}
|
|
197
197
|
const store = await service.getStore();
|
|
198
198
|
const counts = store ? await store.countScoped() : { local: 0, global: 0, projects: 0 };
|
|
199
|
+
const retrieval = getConfig(service.getCwd()).KNOWLEDGE_STORE_RETRIEVAL;
|
|
199
200
|
const embedder = await service.getEmbedder();
|
|
200
201
|
|
|
202
|
+
if (retrieval === 'bm25') {
|
|
203
|
+
// Lexical mode: no embedding model exists — null is the healthy state, not
|
|
204
|
+
// a failure. Report BM25-specific diagnostics instead of the embedder's.
|
|
205
|
+
return {
|
|
206
|
+
healthy: true,
|
|
207
|
+
diagnostic: {
|
|
208
|
+
status: 'initialized',
|
|
209
|
+
retrieval: 'bm25',
|
|
210
|
+
model: 'none (lexical BM25)',
|
|
211
|
+
localEntries: counts.local,
|
|
212
|
+
globalEntries: counts.global,
|
|
213
|
+
totalProjects: counts.projects
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
201
218
|
if (!embedder) {
|
|
202
219
|
return { healthy: false, error: 'Embedder service not available' };
|
|
203
220
|
}
|
|
@@ -240,6 +257,7 @@ export function registerHealthChecks(registry: IHealthRegistryService, container
|
|
|
240
257
|
healthy: true,
|
|
241
258
|
diagnostic: {
|
|
242
259
|
status: 'initialized',
|
|
260
|
+
retrieval,
|
|
243
261
|
device,
|
|
244
262
|
model: config.EMBEDDING_MODEL,
|
|
245
263
|
localEntries: counts.local,
|