@lincoln504/pi-research 0.1.12 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +64 -115
  3. package/dist/cli.mjs +23211 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +102 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +213 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +113 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +64 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1044 -0
  30. package/src/config.ts +1036 -147
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +264 -7
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -174
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -577
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
package/docs/SDK.md ADDED
@@ -0,0 +1,113 @@
1
+ ## SDK
2
+
3
+ The programmatic SDK for scripts, CI, and custom tooling. For configuration — the
4
+ layering model, the TUI settings, and every environment variable — see
5
+ [CONFIGURATION.md](CONFIGURATION.md).
6
+
7
+ `src/sdk.ts` is a library for scripts, CI, and custom tooling. It is configured
8
+ from code, not from a global overlay file — there is no `sdk.env`. It reads the
9
+ base `~/.pi/research/config.env` as a baseline, and everything is
10
+ overridable via `options.config`. Pass `ignoreGlobalConfig: true` to ignore the
11
+ global file entirely and run purely from defaults + `process.env` + `options.config`
12
+ — self-contained and reproducible from code.
13
+
14
+ > Runtime requirement. The package exports (`.` and `/sdk`) resolve to
15
+ > TypeScript source — there is no transpiled `dist/sdk.js`. It must run on a
16
+ > runtime that *transforms* TypeScript, not merely strips types: the source uses
17
+ > `enum` and constructor parameter properties, which Node's strip-only mode
18
+ > (`--experimental-strip-types`, the default since Node 23.6) rejects with
19
+ > `ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX`. Use one of:
20
+ > - the pi host, which loads it natively;
21
+ > - **`node --experimental-transform-types your-script.ts`** — must be passed
22
+ > explicitly; full transform is not the default on any current Node release;
23
+ > - a loader such as `tsx` or `ts-node`.
24
+ >
25
+ > Plain `node script.js` doing `require('@lincoln504/pi-research/sdk')`, or
26
+ > `--experimental-strip-types`, will not work. (`engines.node` is `>=22.19.0`.)
27
+
28
+ ```typescript
29
+ import {
30
+ initResearchSDK,
31
+ runDeepResearch,
32
+ runQuickResearch,
33
+ getResearchReports,
34
+ shutdownResearchSDK,
35
+ } from '@lincoln504/pi-research';
36
+
37
+ // 1. Initialize (configured entirely in code; no global config needed)
38
+ await initResearchSDK({
39
+ model: 'openrouter/deepseek/deepseek-v4-flash', // string "provider/id" or a Model object
40
+ ignoreGlobalConfig: true, // hermetic: ignore ~/.pi/research/config.env
41
+ config: { MAX_SCRAPE_BATCHES: 4 }, // typed Config overrides
42
+ });
43
+
44
+ // 2. Deep research (depth 1–3)
45
+ const markdown = await runDeepResearch('solid-state battery technology', { depth: 2 });
46
+
47
+ // 3. Quick research (depth 0)
48
+ const quick = await runQuickResearch('what is the capital of France');
49
+
50
+ // 4. Retrieve per-researcher reports from the last run
51
+ const reports = await getResearchReports();
52
+
53
+ // 5. Cleanup — REQUIRED: drains the writer queue, closes LanceDB, kills workers
54
+ await shutdownResearchSDK();
55
+ ```
56
+
57
+ `initResearchSDK` must run before any research call. Auth resolves from
58
+ `options.apiKey` + `options.provider`, else `process.env.PI_RESEARCH_API_KEY` /
59
+ `PI_RESEARCH_PROVIDER`, else pi's `~/.pi/agent/auth.json`. Other exports include
60
+ `runResearchDetailed`, `searchKnowledge`, `scrapeUrl`, `getResearchHealth`,
61
+ `getLastRunStats`, and `getSessionMetrics`, plus `exportKnowledge` (write the
62
+ knowledge store to a web-consumable JSON file) and the post-run telemetry accessors
63
+ `getLastRunMetrics`, `getLastRunSummary`, and `getLastErrorReport`. Both
64
+ `@lincoln504/pi-research` and `@lincoln504/pi-research/sdk` export these symbols.
65
+
66
+ > Concurrency: a single initialized SDK instance runs one research call at a time.
67
+ > Overlapping `runDeepResearch`/`runQuickResearch` calls on the same instance throw
68
+ > — run them sequentially, or use a separate process per concurrent run.
69
+
70
+ The SDK does not write report files. Report export is a front-end concern — the pi
71
+ extension and the CLI / agent skill do it when `PI_RESEARCH_REPORT_EXPORT_ENABLED=true`.
72
+
73
+ ### Init options
74
+
75
+ | Option | Description |
76
+ |--------|-------------|
77
+ | `model` | `"provider/id"` string or a `Model` object. Omit to use the first available pi model. |
78
+ | `apiKey` / `provider` | Explicit credentials (provider required with apiKey). |
79
+ | `config` | `Partial<Config>` overrides, applied over the base/defaults. |
80
+ | `ignoreGlobalConfig` | Skip `config.env` entirely — defaults + `process.env` + `config` only. |
81
+ | `cwd` | Working directory for logs and the knowledge store. |
82
+ | `verbose` | Mirror logs to the console. |
83
+
84
+ For configuration precedence, the per-front-end overlays, and the full
85
+ environment-variable reference, see [CONFIGURATION.md](CONFIGURATION.md).
86
+
87
+ ### Health & knowledge-store APIs
88
+
89
+ The `health` tool (and the SDK's `getResearchHealth()`) runs every registered
90
+ health check — browser capability, browser runtime, knowledge store, and state
91
+ manager — and returns a structured report:
92
+
93
+ ```typescript
94
+ import { initResearchSDK, getResearchHealth } from '@lincoln504/pi-research/sdk';
95
+
96
+ await initResearchSDK(); // required first — throws if not initialized
97
+ const result = await getResearchHealth();
98
+ // { success: boolean, status: 'healthy' | 'degraded' | 'unhealthy', components: [...] }
99
+ ```
100
+
101
+ The knowledge store is an internal service, not a public export. It is populated
102
+ automatically during research runs; query stored findings with the SDK's
103
+ `searchKnowledge()` or the `research_knowledge_search` tool. The vector dimension
104
+ is model-dependent (auto-detected); stored fields are `text`, `content`,
105
+ `vector`, `url`, `metadata`, `timestamp`, `workspace`, `is_global`, and
106
+ `ingestion_type`.
107
+
108
+ ### Example Use
109
+
110
+ The [Wall of Shame](https://wallofshame.io) project ([repo](https://github.com/Lincoln504/wall-of-shame))
111
+ uses this SDK in its agent pipeline: it calls `initResearchSDK` and the research
112
+ entry points (`runQuickResearch` / `runDeepResearch`) per investigation, and uses
113
+ the `scrapeUrl`, `verifyUrl`, and `repairJson` exports directly.
@@ -0,0 +1,64 @@
1
+ ## Third-Party Notices
2
+
3
+ `@lincoln504/pi-research` is licensed under the MIT License (see `LICENSE`). That
4
+ license covers this project's own source code only.
5
+
6
+ This package depends on third-party software distributed under its own license
7
+ terms. Dependencies are resolved and installed by npm at install time (by
8
+ reference); they are **not** bundled into this package's published artifacts
9
+ (`dist/*` is built with esbuild `--packages=external`, so no third-party code is
10
+ inlined).
11
+
12
+ ### Notable dependency licensing
13
+
14
+ camoufox-js / Camoufox browser
15
+
16
+ The `camoufox-js` npm package itself is licensed under **MPL-2.0** (file-level
17
+ copyleft). It is resolved at install time and **not bundled** into this package's
18
+ artifacts (`dist/*` is built `--packages=external`), so no MPL-covered source is
19
+ combined into or redistributed by this MIT package. It is used in-process only to
20
+ configure and launch the Camoufox (Firefox-fork) browser **binary as a separate
21
+ operating-system process**, which this package then drives over the Playwright/CDP
22
+ protocol. The Camoufox binary is downloaded and run separately and is subject to its
23
+ own license.
24
+
25
+ ua-parser-js
26
+
27
+ `ua-parser-js` 2.x is licensed under AGPL-3.0-or-later. To keep this MIT-licensed
28
+ package free of AGPL-licensed code in its dependency tree, `package.json`
29
+ `overrides` pins `ua-parser-js` to the `1.x` line, which is MIT-licensed. The
30
+ 1.x and 2.x APIs are compatible for the only surface our transitive dependency
31
+ (`camoufox-js`) uses, and the pin is behavior-neutral for this package's usage
32
+ (no custom `navigator.userAgent`/`fingerprint` is passed to Camoufox).
33
+
34
+ youtubei.js / bgutils-js / jsdom (YouTube transcripts)
35
+
36
+ The `youtube_transcript` tool depends on `youtubei.js` (MIT), `bgutils-js` (MIT),
37
+ and `jsdom` (MIT). All three are standard MIT and are resolved at install time,
38
+ not bundled. `youtubei.js` and `bgutils-js` interact with YouTube's **unofficial**
39
+ InnerTube API and BotGuard attestation to fetch publicly available captions
40
+ without an API key; no credentials are used and only caption text is read.
41
+ Operators are responsible for ensuring their use complies with YouTube's Terms of
42
+ Service. `jsdom` provides the DOM environment the BotGuard VM requires; it is
43
+ loaded lazily (only when the tool runs) and never at startup.
44
+
45
+ sharp / @img/sharp-libvips (via @huggingface/transformers)
46
+
47
+ `@huggingface/transformers` pulls in `sharp` (Apache-2.0) for image decoding, which
48
+ in turn installs prebuilt `@img/sharp-libvips-*` native packages licensed under
49
+ **LGPL-3.0-or-later**. These are platform-specific shared libraries linked
50
+ **dynamically** at runtime and resolved at install time; no LGPL code is statically
51
+ linked into or bundled with this package (`dist/*` is built `--packages=external`).
52
+ Dynamic linkage of an unmodified LGPL library from an MIT-licensed program is
53
+ permitted by the LGPL. The image path is only exercised by the embedding/knowledge
54
+ stack; text-only research does not load it.
55
+
56
+ ### Verifying the dependency tree
57
+
58
+ ```
59
+ npm ls ua-parser-js # expect ua-parser-js@1.x (overridden, MIT)
60
+ ```
61
+
62
+ For a full license inventory of installed dependencies, run a license scanner of
63
+ your choice (e.g. `npx license-checker --summary`) against your installed
64
+ `node_modules`.
package/package.json CHANGED
@@ -1,18 +1,43 @@
1
1
  {
2
2
  "name": "@lincoln504/pi-research",
3
- "version": "0.1.12",
3
+ "version": "1.0.0",
4
4
  "description": "A set of web research tools (search, scrape, and others) and research agent system that uses them safely and efficiently.",
5
5
  "type": "module",
6
6
  "private": false,
7
7
  "main": "./src/index.ts",
8
+ "types": "./src/index.ts",
9
+ "bin": {
10
+ "pi-research": "./dist/cli.mjs"
11
+ },
12
+ "exports": {
13
+ ".": "./src/index.ts",
14
+ "./sdk": "./src/sdk.ts",
15
+ "./package.json": "./package.json"
16
+ },
8
17
  "files": [
9
18
  "src/",
10
- "config/",
11
- "docs/",
12
- "scripts/",
19
+ "dist/cli.mjs",
20
+ "dist/thread-worker.mjs",
21
+ "dist/webgpu-probe.mjs",
22
+ "dist/prompts/",
23
+ "skills/",
24
+ "!skills/pi-research/scripts/run.ts",
25
+ "!src/.env",
26
+ "!src/.env.local",
27
+ "!src/.env.*.local",
28
+ "docs/PI-EXTENSION.md",
29
+ "docs/SDK.md",
30
+ "docs/CONFIGURATION.md",
31
+ "docs/KNOWLEDGE-STORE.md",
32
+ "docs/AGENT-SKILL.md",
33
+ "docs/ARCHITECTURE.md",
34
+ "scripts/setup.cjs",
35
+ "scripts/cleanup.cjs",
36
+ "scripts/build.cjs",
37
+ "scripts/prepare.cjs",
13
38
  "LICENSE",
14
- "README.md",
15
- "tsconfig.json"
39
+ "docs/THIRD-PARTY-NOTICES.md",
40
+ "README.md"
16
41
  ],
17
42
  "repository": {
18
43
  "type": "git",
@@ -26,67 +51,108 @@
26
51
  "pi",
27
52
  "pi-package",
28
53
  "pi-extension",
54
+ "openclaw",
55
+ "agent-skill",
29
56
  "research",
30
57
  "agent",
31
58
  "web-search",
32
59
  "scraping",
60
+ "stealth-browser",
61
+ "camoufox",
33
62
  "security",
34
63
  "stack-exchange"
35
64
  ],
36
- "author": "Lincolndeen",
65
+ "author": "Lincoln Deen",
37
66
  "contributors": [
38
67
  "MuratAitov"
39
68
  ],
40
69
  "license": "MIT",
70
+ "publishConfig": {
71
+ "access": "public"
72
+ },
41
73
  "engines": {
42
- "node": "^22.13.0 || >=24.0.0"
74
+ "node": ">=22.19.0"
43
75
  },
44
76
  "scripts": {
45
77
  "lint": "npx eslint --config config/tooling/eslint.config.js src/",
46
78
  "lint:fix": "npx eslint --config config/tooling/eslint.config.js src/ --fix",
47
79
  "type-check": "tsc --noEmit",
48
- "test": "vitest run --config config/tooling/vitest.config.unit.ts",
80
+ "type-check:tests": "tsc --noEmit --project config/tooling/tsconfig.test.json",
81
+ "build:worker": "node scripts/build.cjs worker",
82
+ "test": "npm run test:unit",
49
83
  "test:unit": "vitest run --config config/tooling/vitest.config.unit.ts",
50
- "test:integration": "vitest run --config config/tooling/vitest.config.integration.ts",
84
+ "test:integration": "npm run test:integration:parallel && npm run test:integration:serial",
85
+ "test:integration:parallel": "vitest run --config config/tooling/vitest.config.integration.parallel.ts",
86
+ "test:integration:serial": "vitest run --config config/tooling/vitest.config.integration.serial.ts",
87
+ "test:all": "npm run test:unit && npm run test:integration",
51
88
  "test:watch": "vitest --config config/tooling/vitest.config.unit.ts",
52
- "test:coverage": "vitest run --config config/tooling/vitest.config.ts --coverage",
53
- "postinstall": "node scripts/setup.js",
54
- "install:browsers": "npx playwright install chromium",
55
- "install:deps": "npx playwright install-deps",
56
- "setup": "npm install && npm run install:browsers"
89
+ "test:coverage": "vitest run --config config/tooling/vitest.config.unit.ts --coverage",
90
+ "build:cli": "node scripts/build.cjs cli",
91
+ "build:skill": "node scripts/build.cjs skill",
92
+ "build": "node scripts/build.cjs all",
93
+ "prepublishOnly": "node scripts/build.cjs all && node scripts/verify-package.cjs manifest",
94
+ "prepare": "node scripts/prepare.cjs",
95
+ "version": "node -e \"const fs=require('fs');const p=require('./package.json');const sp='./skills/pi-research/SKILL.md';let s=fs.readFileSync(sp,'utf8');s=s.replace(/\\\"version\\\": \\\"[^\\\"]*\\\"/,'\\\"version\\\": \\\"'+p.version+'\\\"');fs.writeFileSync(sp,s);console.log('✔ synced SKILL.md to v'+p.version)\" && git add skills/pi-research/SKILL.md",
96
+ "postinstall": "node scripts/setup.cjs",
97
+ "preuninstall": "node scripts/cleanup.cjs",
98
+ "setup": "node scripts/setup.cjs",
99
+ "install:system-deps": "node scripts/setup.cjs --system-deps",
100
+ "deps:check": "depcruise --config config/tooling/dependency-cruiser.cjs --output-type err-long src",
101
+ "deps:generate": "npx --yes madge --image docs/deps.svg src/index.ts",
102
+ "analyze:dependencies": "ncu",
103
+ "analyze:compatibility": "depngn"
57
104
  },
58
105
  "pi": {
59
106
  "extensions": [
60
107
  "./src/index.ts"
61
- ]
108
+ ],
109
+ "image": "https://raw.githubusercontent.com/Lincoln504/pi-research/main/docs/media/hero.png"
62
110
  },
63
111
  "dependencies": {
64
- "@kreuzberg/html-to-markdown-node": "^2.30.0",
65
- "dockerode": "^4.0.10",
66
- "js-yaml": "^4.1.1",
112
+ "@earendil-works/pi-ai": ">=0.80.2 <1",
113
+ "@earendil-works/pi-coding-agent": ">=0.80.2 <1",
114
+ "@earendil-works/pi-tui": ">=0.80.2 <1",
115
+ "@huggingface/transformers": "4.2.0",
116
+ "@kreuzberg/html-to-markdown-node": "^3.5.5",
117
+ "@lancedb/lancedb": "0.29.0",
118
+ "apache-arrow": "21.1.0",
119
+ "bgutils-js": "^3.2.0",
120
+ "camoufox-js": "^0.10.2",
121
+ "esbuild": "^0.28.1",
122
+ "impit": "0.13.0",
123
+ "jsdom": "^29.1.1",
67
124
  "node-html-markdown": "^2.0.0",
68
- "playwright": "^1.52.0"
125
+ "pdf-oxide-wasm": "^0.3.70",
126
+ "playwright-core": "1.60.0",
127
+ "poolifier": "^5.3.2",
128
+ "typebox": "1.1.38",
129
+ "undici": "^8.6.0",
130
+ "youtubei.js": "^17.2.0"
69
131
  },
70
132
  "optionalDependencies": {
71
- "@kreuzberg/html-to-markdown-node-darwin-arm64": "^2.30.0",
72
- "@kreuzberg/html-to-markdown-node-darwin-x64": "^2.30.0",
73
- "@kreuzberg/html-to-markdown-node-win32-x64-msvc": "^2.30.0"
133
+ "webgpu": "^0.4.0"
74
134
  },
75
- "peerDependencies": {
76
- "@mariozechner/pi-coding-agent": ">=0.65.0",
77
- "@sinclair/typebox": ">=0.34.0"
135
+ "overrides": {
136
+ "lodash.isequal": "npm:fast-deep-equal@^3.1.3",
137
+ "ip-address": "^10.2.0",
138
+ "apache-arrow": "21.1.0",
139
+ "onnxruntime-web": "1.24.3",
140
+ "node-domexception": "npm:@nolyfill/domexception@^1.0.28",
141
+ "ua-parser-js": "^1.0.41",
142
+ "protobufjs": "7.6.4",
143
+ "vite": "^8.0.16"
78
144
  },
79
145
  "devDependencies": {
80
146
  "@eslint/js": "^10.0.1",
81
- "@types/dockerode": "^4.0.1",
82
- "@types/js-yaml": "^4.0.9",
83
- "@typescript-eslint/eslint-plugin": "^8.58.0",
84
- "@typescript-eslint/parser": "^8.58.0",
85
- "@vitest/coverage-v8": "^4.1.2",
86
- "eslint": "^10.1.0",
87
- "eslint-config-prettier": "^10.1.0",
88
- "testcontainers": "^11.13.0",
89
- "typescript": "^6.0.2",
90
- "vitest": "^4.1.2"
147
+ "@types/jsdom": "^28.0.3",
148
+ "@typescript-eslint/eslint-plugin": "^8.60.0",
149
+ "@typescript-eslint/parser": "^8.60.0",
150
+ "@vitest/coverage-v8": "^4.1.7",
151
+ "dependency-cruiser": "^17.4.2",
152
+ "depngn": "^0.6.0",
153
+ "eslint": "^10.4.0",
154
+ "npm-check-updates": "^22.2.1",
155
+ "typescript": ">=6.0.3 <6.1.0",
156
+ "vitest": "^4.1.7"
91
157
  }
92
158
  }
@@ -0,0 +1,140 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * Cross-platform build of all distribution artifacts via the esbuild JS API.
5
+ *
6
+ * Using the JS API (not shell `esbuild ...` invocations) removes every
7
+ * shell-quoting hazard between POSIX `sh` and Windows `cmd.exe` — most notably
8
+ * the `--banner:js='#!/usr/bin/env node'` shebang, whose single quotes are not
9
+ * grouping characters in cmd.exe and so split the build on the space. The
10
+ * banner is just a JS string here, identical on every platform.
11
+ *
12
+ * Targets (dispatch by argv, or build all when no/`all` arg):
13
+ * worker -> src/infrastructure/browser/thread-worker.mjs (browser worker)
14
+ * cli -> dist/cli.mjs (+ dist/prompts, dist/thread-worker.mjs, dist/webgpu-probe.mjs)
15
+ * skill -> skills/pi-research/scripts/run.mjs (skill launcher, shebang)
16
+ *
17
+ * Note: `packages: 'external'` already externalizes every bare/`node:` import,
18
+ * so the old `--external:node:*` flag was redundant — output is identical.
19
+ */
20
+
21
+ 'use strict';
22
+
23
+ const esbuild = require('esbuild');
24
+ const fs = require('fs');
25
+ const path = require('path');
26
+
27
+ const ROOT = path.join(__dirname, '..');
28
+ const SHEBANG = '#!/usr/bin/env node';
29
+
30
+ const COMMON = {
31
+ bundle: true,
32
+ platform: 'node',
33
+ format: 'esm',
34
+ packages: 'external',
35
+ logLevel: 'warning',
36
+ };
37
+
38
+ const p = (...segs) => path.join(ROOT, ...segs);
39
+
40
+ // The out-of-process WebGPU viability probe is plain .mjs (not bundled). It must
41
+ // sit next to the bundled entry points so the embedder can resolve it relative to
42
+ // import.meta.url, and so it resolves @huggingface/transformers from the installed
43
+ // package's node_modules. The bundled dist/cli.mjs depends on it.
44
+ function copyWebGpuProbe() {
45
+ fs.mkdirSync(p('dist'), { recursive: true });
46
+ fs.copyFileSync(
47
+ p('src', 'knowledge', 'webgpu-probe.mjs'),
48
+ p('dist', 'webgpu-probe.mjs'),
49
+ );
50
+ }
51
+
52
+ // Build the browser worker bundle. Factored out so the cli target can guarantee
53
+ // it exists when built in isolation (`build:cli`), not only via `build all`.
54
+ function buildWorker() {
55
+ return esbuild.build({
56
+ ...COMMON,
57
+ entryPoints: [p('src', 'infrastructure', 'browser', 'thread-worker.ts')],
58
+ outfile: p('src', 'infrastructure', 'browser', 'thread-worker.mjs'),
59
+ });
60
+ }
61
+
62
+ async function ensureWorkerBuilt() {
63
+ const src = p('src', 'infrastructure', 'browser', 'thread-worker.ts');
64
+ const out = p('src', 'infrastructure', 'browser', 'thread-worker.mjs');
65
+ // Rebuild if the bundle is missing OR older than its source, so an isolated
66
+ // `build:cli` after editing thread-worker.ts never ships a stale worker.
67
+ const stale =
68
+ !fs.existsSync(out) || fs.statSync(out).mtimeMs < fs.statSync(src).mtimeMs;
69
+ if (stale) {
70
+ await buildWorker();
71
+ }
72
+ }
73
+
74
+ // Copy the built worker bundle into dist/. dist/thread-worker.mjs is resolved by
75
+ // worker-pool-manager.ts at runtime, so the bundled cli — which spins up the
76
+ // browser pool — needs it next to its bundle.
77
+ function copyThreadWorker() {
78
+ fs.mkdirSync(p('dist'), { recursive: true });
79
+ fs.copyFileSync(
80
+ p('src', 'infrastructure', 'browser', 'thread-worker.mjs'),
81
+ p('dist', 'thread-worker.mjs'),
82
+ );
83
+ }
84
+
85
+ // Copy the prompt templates into dist/prompts/. The bundled dist/cli.mjs resolves
86
+ // prompts relative to its own location (loadPrompt → join(dirname(cli.mjs),
87
+ // 'prompts')), so the templates must sit next to the bundle.
88
+ function copyPrompts() {
89
+ fs.mkdirSync(p('dist', 'prompts'), { recursive: true });
90
+ for (const f of fs.readdirSync(p('src', 'prompts'))) {
91
+ fs.copyFileSync(p('src', 'prompts', f), p('dist', 'prompts', f));
92
+ }
93
+ }
94
+
95
+ const TARGETS = {
96
+ worker: () => buildWorker(),
97
+ cli: async () => {
98
+ await ensureWorkerBuilt();
99
+ await esbuild.build({
100
+ ...COMMON,
101
+ entryPoints: [p('src', 'cli.ts')],
102
+ outfile: p('dist', 'cli.mjs'),
103
+ banner: { js: SHEBANG },
104
+ });
105
+ copyThreadWorker();
106
+ copyWebGpuProbe();
107
+ copyPrompts();
108
+ },
109
+ skill: () =>
110
+ esbuild.build({
111
+ ...COMMON,
112
+ entryPoints: [p('skills', 'pi-research', 'scripts', 'run.ts')],
113
+ outfile: p('skills', 'pi-research', 'scripts', 'run.mjs'),
114
+ banner: { js: SHEBANG },
115
+ }),
116
+ };
117
+
118
+ // The cli target's copyThreadWorker step depends on the worker output existing,
119
+ // so the full build runs worker before cli; skill is independent.
120
+ const ALL_ORDER = ['worker', 'cli', 'skill'];
121
+
122
+ async function main() {
123
+ const arg = process.argv[2];
124
+ const targets = !arg || arg === 'all' ? ALL_ORDER : [arg];
125
+
126
+ for (const name of targets) {
127
+ const build = TARGETS[name];
128
+ if (!build) {
129
+ console.error(`[build] Unknown target: ${name}`);
130
+ console.error(`[build] Valid targets: ${Object.keys(TARGETS).join(', ')}, all`);
131
+ process.exit(1);
132
+ }
133
+ await build();
134
+ }
135
+ }
136
+
137
+ main().catch((err) => {
138
+ console.error('[build] Build failed:', err && err.message ? err.message : err);
139
+ process.exit(1);
140
+ });
@@ -0,0 +1,149 @@
1
+ #!/usr/bin/env node
2
+
3
+ const { execSync } = require('child_process');
4
+ const { rmSync, existsSync, lstatSync, readlinkSync, readFileSync, writeFileSync, unlinkSync } = require('fs');
5
+ const os = require('os');
6
+ const path = require('path');
7
+
8
+ // __dirname is a built-in global in CommonJS (.cjs) modules
9
+ const projectRoot = path.join(__dirname, '..');
10
+
11
+ /**
12
+ * Remove research-skill installs (symlinks/copies) this package created in the
13
+ * coding-agent harnesses, so `npm uninstall` is symmetric with the in-app
14
+ * installer (the /research-config "Install Skill in Coding Agents" action).
15
+ * Reads the manifest written by the installer and removes ONLY entries we still
16
+ * own (a symlink pointing into a pi-research skills dir, or a copy carrying our
17
+ * package marker). Foreign directories are never touched. Best-effort; never
18
+ * throws (preuninstall must not fail the uninstall).
19
+ */
20
+ function removeInstalledSkills() {
21
+ const manifestPath = path.join(os.homedir(), '.pi', 'research', 'installed-skills.json');
22
+ // Read directly and let the catch handle a missing/unreadable manifest. Avoiding
23
+ // a separate existsSync() check removes a check-then-use (TOCTOU) on the file that
24
+ // is later rewritten below — the read itself is the only gate we need.
25
+ let manifest;
26
+ try {
27
+ manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
28
+ } catch {
29
+ return;
30
+ }
31
+ const entries = Array.isArray(manifest && manifest.entries) ? manifest.entries : [];
32
+ const kept = [];
33
+ for (const e of entries) {
34
+ const p = e && e.path;
35
+ if (!p) continue;
36
+ // Use lstat, not existsSync: existsSync follows symlinks, so a DANGLING
37
+ // symlink (our target already removed) reports absent and we'd drop the
38
+ // manifest entry while leaving the broken symlink on disk. lstat sees the
39
+ // link itself.
40
+ let lst;
41
+ try { lst = lstatSync(p); } catch { continue; } // truly gone → drop entry
42
+ let owned = false;
43
+ try {
44
+ if (lst.isSymbolicLink()) {
45
+ const dest = path.resolve(path.dirname(p), readlinkSync(p));
46
+ owned = /[/\\]pi-research[/\\]skills[/\\]pi-research$/.test(dest);
47
+ } else {
48
+ const md = readFileSync(path.join(p, 'SKILL.md'), 'utf-8');
49
+ owned = md.includes('@lincoln504/pi-research');
50
+ }
51
+ } catch { /* treat as not-owned */ }
52
+ if (!owned) { kept.push(e); continue; }
53
+ try {
54
+ if (lstatSync(p).isSymbolicLink()) unlinkSync(p);
55
+ else rmSync(p, { recursive: true, force: true });
56
+ console.log(`pi-research: removed skill install at ${p}`);
57
+ } catch (err) {
58
+ kept.push(e);
59
+ console.warn(`pi-research: could not remove skill install ${p}: ${err instanceof Error ? err.message : String(err)}`);
60
+ }
61
+ }
62
+ try {
63
+ if (kept.length > 0) {
64
+ writeFileSync(manifestPath, JSON.stringify({ version: 1, package: '@lincoln504/pi-research', entries: kept }, null, 2) + '\n', 'utf-8');
65
+ } else {
66
+ unlinkSync(manifestPath);
67
+ }
68
+ } catch { /* best effort */ }
69
+ }
70
+
71
+ removeInstalledSkills();
72
+
73
+ /**
74
+ * Resolve the camoufox-js binary path.
75
+ *
76
+ * See setup.cjs for rationale — dependencies are hoisted to root node_modules.
77
+ */
78
+ function resolveCamoufoxBin() {
79
+ try {
80
+ const pkgDir = path.dirname(require.resolve('camoufox-js/package.json', { paths: [projectRoot] }));
81
+ const binDir = path.join(pkgDir, '..', '.bin');
82
+ // On Windows the executable is the .cmd shim; see setup.cjs.
83
+ const bin = path.join(binDir, process.platform === 'win32' ? 'camoufox-js.cmd' : 'camoufox-js');
84
+ if (existsSync(bin)) return bin;
85
+ } catch (_) { /* fall through */ }
86
+ return null;
87
+ }
88
+
89
+ // Remove camoufox browser binaries — only when explicitly requested.
90
+ // The shared cache (~/.cache/camoufox) may be used by other tools, so we do NOT
91
+ // purge it by default. Set PI_RESEARCH_PURGE_BROWSERS=1 to opt in.
92
+ if (process.env.PI_RESEARCH_PURGE_BROWSERS === '1') {
93
+ if (process.env.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD !== '1') {
94
+ try {
95
+ const bin = resolveCamoufoxBin();
96
+ const cmd = bin ? `"${bin}" remove` : 'npx camoufox-js remove';
97
+ execSync(cmd, { stdio: 'inherit' });
98
+ console.log('pi-research: camoufox browser binaries removed.');
99
+ } catch (error) {
100
+ console.warn(`pi-research: could not remove camoufox binaries: ${error instanceof Error ? error.message : String(error)}`);
101
+ console.warn('pi-research: to remove manually, run: npx camoufox-js remove');
102
+ }
103
+ }
104
+ } else {
105
+ console.log('pi-research: leaving shared camoufox binaries in place (set PI_RESEARCH_PURGE_BROWSERS=1 to remove).');
106
+ }
107
+
108
+ // Remove legacy project-local browser cache if present
109
+ const legacyCacheDir = path.join(projectRoot, '.browser');
110
+ if (existsSync(legacyCacheDir)) {
111
+ try {
112
+ rmSync(legacyCacheDir, { recursive: true, force: true });
113
+ } catch (error) {
114
+ console.warn(`pi-research: could not remove ${legacyCacheDir}: ${error instanceof Error ? error.message : String(error)}`);
115
+ }
116
+ }
117
+
118
+ /**
119
+ * Purge pi-research's OWN scratch dirs on uninstall — there is no next startup to
120
+ * reclaim them, so they would otherwise persist forever. Both are exclusively ours
121
+ * (named `pi-research`), so removal is safe. User config (~/.pi/research/config.env)
122
+ * and the knowledge_db are deliberately preserved. Best-effort; never throws.
123
+ * Paths mirror the runtime resolution (config.ts / state-path-configuration.ts) and
124
+ * honor the same overrides, so this stays correct cross-platform.
125
+ */
126
+ function removeOwnedScratchDir(label, dir) {
127
+ if (!dir || !existsSync(dir)) return;
128
+ try {
129
+ rmSync(dir, { recursive: true, force: true });
130
+ console.log(`pi-research: removed ${label} ${dir}`);
131
+ } catch (error) {
132
+ console.warn(`pi-research: could not remove ${label} ${dir}: ${error instanceof Error ? error.message : String(error)}`);
133
+ }
134
+ }
135
+
136
+ // Cache tree: browser profiles, webgpu-viability probe, downloaded embedding models.
137
+ // Same resolution as config.ts (XDG_CACHE_HOME || ~/.cache, then pi-research) on
138
+ // every platform. Shared camoufox binaries live in ~/.cache/camoufox (handled above).
139
+ const cacheHome = process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache');
140
+ removeOwnedScratchDir('cache dir', path.join(cacheHome, 'pi-research'));
141
+
142
+ // State tree: transient run state, locks, backups, project-settings. Now lives in
143
+ // pi-research's own namespace (~/.pi/research/state), honoring the same overrides
144
+ // as getGlobalConfigDir()/PI_RESEARCH_STATE_DIR.
145
+ const stateDir = process.env.PI_RESEARCH_STATE_DIR
146
+ || path.join(os.homedir(), process.env.PI_RESEARCH_CONFIG_DIR_NAME || '.pi', 'research', 'state');
147
+ removeOwnedScratchDir('state dir', stateDir);
148
+
149
+ process.exit(0);