@lincoln504/pi-research 0.1.13 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +55 -138
  3. package/dist/cli.mjs +23375 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +130 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +211 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +122 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +66 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1195 -0
  30. package/src/config.ts +1032 -155
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +258 -11
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -212
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -644
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
package/scripts/setup.js DELETED
@@ -1,212 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- /**
4
- * pi-research Postinstall Setup Script
5
- *
6
- * This script runs automatically after `npm install` and ensures:
7
- * 1. Playwright browser binaries are installed (Chromium)
8
- * 2. System dependencies are available (if run with --system-deps flag)
9
- *
10
- * Key Design Principles:
11
- * - NEVER exits with code 1 - this would block npm install from completing
12
- * - Uses npx playwright install, which works if playwright is in node_modules
13
- * - Provides clear feedback about installation status
14
- * - Gracefully handles missing dependencies (warns but doesn't fail)
15
- *
16
- * Usage:
17
- * npm run setup # Full setup: install deps + browsers
18
- * npm run install:browsers # Install only browser binaries
19
- * npm run install:deps # Install system dependencies only
20
- *
21
- * Environment Variables:
22
- * PLAYWRIGHT_BROWSERS_PATH - Custom path for browser cache
23
- * PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 - Skip browser download
24
- */
25
-
26
- import { execSync } from 'child_process';
27
- import { existsSync, readdirSync, statSync } from 'fs';
28
- import { dirname, join } from 'path';
29
- import { fileURLToPath } from 'url';
30
- import { homedir, platform } from 'os';
31
-
32
- const __filename = fileURLToPath(import.meta.url);
33
- const __dirname = dirname(__filename);
34
-
35
- console.log('='.repeat(60));
36
- console.log('🔧 pi-research Setup');
37
- console.log('='.repeat(60));
38
-
39
- // Check Node.js version
40
- const nodeVersion = process.version.replace('v', '');
41
- const [major] = nodeVersion.split('.').map(Number);
42
-
43
- console.log(`\nℹ️ Node.js version: v${nodeVersion}`);
44
-
45
- const requiredVersions = {
46
- min: 22,
47
- preferred: [22, 24]
48
- };
49
-
50
- if (major < requiredVersions.min) {
51
- console.warn(
52
- `⚠️ WARNING: Node.js ${nodeVersion} is below the minimum required version ${requiredVersions.min}.`
53
- );
54
- console.warn(' Please upgrade to Node.js 22.13.0+ or 24.x+');
55
- }
56
-
57
- console.log(` Required: >= ${requiredVersions.min}.0.0`);
58
-
59
- // Detect OS for browser installation
60
- const isLinux = platform() === 'linux';
61
- const isDarwin = platform() === 'darwin';
62
- const isWindows = platform() === 'win32';
63
-
64
- console.log(`\n📦 Detected platform: ${platform()}`);
65
-
66
- // Track installation success
67
- let playwrightInstalled = false;
68
- let browsersInstalled = false;
69
- let systemDepsInstalled = false;
70
-
71
- // Check if we should skip browser installation
72
- const skipBrowserDownload = process.env.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD === '1';
73
-
74
- if (skipBrowserDownload) {
75
- console.log('\n⏭️ Skipping browser installation (PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1)');
76
- } else {
77
- // Install browsers using npx playwright
78
- console.log('\n🌐 Installing Playwright browsers...');
79
-
80
- try {
81
- // Check if user wants to install system dependencies
82
- const installDeps = process.argv.includes('--system-deps') ||
83
- process.env.PLAYWRIGHT_INSTALL_DEPS === 'true';
84
-
85
- if (installDeps && isLinux) {
86
- console.log('⚙️ Installing Linux system dependencies for Playwright...');
87
- try {
88
- execSync('npx playwright install-deps', {
89
- stdio: 'inherit',
90
- env: { ...process.env, PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '0' }
91
- });
92
- console.log('✅ System dependencies installed');
93
- systemDepsInstalled = true;
94
- } catch (error) {
95
- console.warn('⚠️ Could not install system dependencies automatically.');
96
- console.warn(' Try running: sudo apt-get install $(npx playwright install-deps --print-deps)');
97
- // Don't fail - continue with browser installation
98
- }
99
- }
100
-
101
- // Install browser binaries
102
- // Only Chromium is needed for scraping (as per scrapers.ts implementation)
103
- console.log(' Installing Chromium browser...');
104
- execSync('npx playwright install chromium', {
105
- stdio: 'inherit',
106
- env: {
107
- ...process.env,
108
- PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH || ''
109
- }
110
- });
111
-
112
- console.log('✅ Browsers installed successfully');
113
- browsersInstalled = true;
114
-
115
- } catch (error) {
116
- // Parse error message to provide better guidance
117
- const errorMsg = error instanceof Error ? error.message : String(error);
118
-
119
- if (errorMsg.includes('npx: command not found')) {
120
- console.warn('\n⚠️ npx not available - skipping browser installation');
121
- console.warn(' Chromium can be installed later with: npm run install:browsers');
122
- } else if (errorMsg.includes('playwright') || errorMsg.includes('ENOENT')) {
123
- console.warn('\n⚠️ Playwright package not yet available - skipping browser installation');
124
- console.warn(' This is normal during initial npm install.');
125
- console.warn(' Chromium will be installed automatically when you run the research tool.');
126
- console.warn(' Or manually with: npm run install:browsers');
127
- } else {
128
- console.warn('\n⚠️ Browser installation failed:');
129
- console.warn(` ${errorMsg}`);
130
- console.warn('\n💡 You can install Chromium later with:');
131
- console.warn(' npm run install:browsers');
132
- }
133
-
134
- // NEVER exit with code 1 - let npm install complete
135
- console.log('\n📦 pi-research installation will continue...');
136
- console.log('='.repeat(60));
137
- console.log('✅ pi-research installed (browsers can be installed later)');
138
- console.log('='.repeat(60) + '\n');
139
- process.exit(0);
140
- }
141
- }
142
-
143
- // Verify installation
144
- console.log('\n📋 Verifying browser installation...');
145
-
146
- const browsers = ['chromium']; // Only Chromium is required for scraping
147
- const cacheDir = process.env['PLAYWRIGHT_BROWSERS_PATH'] ||
148
- (process.platform === 'win32'
149
- ? join(process.env['LOCALAPPDATA'] || join(homedir(), 'AppData', 'Local'), 'ms-playwright')
150
- : join(homedir(), '.cache', 'ms-playwright'));
151
-
152
- browsers.forEach(browser => {
153
- let browserDirs = [];
154
- try {
155
- browserDirs = readdirSync(cacheDir).filter(f =>
156
- f.startsWith(browser) && statSync(join(cacheDir, f)).isDirectory()
157
- );
158
- } catch {
159
- // cache dir may not exist yet or may be at a different path
160
- }
161
-
162
- if (browserDirs.length > 0) {
163
- console.log(` ✅ ${browser}: ${browserDirs.sort().join(', ')}`);
164
- } else {
165
- console.warn(` ⚠️ ${browser}: Not found in cache`);
166
- }
167
- });
168
-
169
- // Show browser versions
170
- console.log('\n🔍 Browser versions:');
171
- try {
172
- const playwrightVersion = execSync('npx playwright --version', { encoding: 'utf8' }).trim();
173
- console.log(` Playwright: ${playwrightVersion}`);
174
- playwrightInstalled = true;
175
- } catch (error) {
176
- console.warn(' Could not determine Playwright version');
177
- }
178
-
179
- // Final status message
180
- const setupComplete = browsersInstalled && playwrightInstalled;
181
-
182
- console.log('\n' + '='.repeat(60));
183
- if (setupComplete) {
184
- console.log('🎉 Setup complete! Chromium browser installed. pi-research is ready to use.');
185
- } else {
186
- console.log('📦 pi-research installed successfully!');
187
- console.log(' Chromium browser installation may be in progress or was skipped.');
188
- console.log(' The browser will be installed automatically when you first use the research tool.');
189
- }
190
- console.log('='.repeat(60));
191
-
192
- // Additional info
193
- console.log('\n📚 Quick start:');
194
- console.log(' • npm test - Run unit tests');
195
- console.log(' • npm run type-check - Type check TypeScript');
196
- console.log(' • npm run lint - Run ESLint');
197
- console.log(' • npm run setup - Full setup with system deps');
198
- console.log(' • npm run install:browsers - Install Chromium browser only');
199
-
200
- // Platform-specific tips
201
- if (isLinux) {
202
- console.log('\n💡 Linux tip: If you encounter issues, install system deps:');
203
- console.log(' sudo apt-get install -y libgbm1 libnss3 libatk1.0-0');
204
- console.log(' libatk-bridge2.0-0 libcups2 libxkbcommon0 libxcomposite1');
205
- } else if (isDarwin) {
206
- console.log('\n💡 macOS: Ensure Xcode Command Line Tools are installed:');
207
- console.log(' xcode-select --install');
208
- } else if (isWindows) {
209
- console.log('\n💡 Windows: Ensure you have the latest Visual C++ redistributables');
210
- }
211
-
212
- console.log('\n');