@lincoln504/pi-research 0.1.12 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +64 -115
  3. package/dist/cli.mjs +23211 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +102 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +213 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +113 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +64 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1044 -0
  30. package/src/config.ts +1036 -147
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +264 -7
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -174
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -577
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -1,129 +0,0 @@
1
- /**
2
- * Deep Research Reducer
3
- *
4
- * PURE FUNCTION logic for state transitions.
5
- * Contains 0 side effects. 100% testable without mocks.
6
- */
7
-
8
- import type { SystemResearchState, DeepResearchEvent } from './deep-research-types.ts';
9
-
10
- export function deepResearchReducer(state: SystemResearchState, event: DeepResearchEvent): SystemResearchState {
11
- // Deep clone state to ensure true purity
12
- const newState: SystemResearchState = {
13
- ...state,
14
- lastUpdated: Date.now(),
15
- initialAgenda: [...state.initialAgenda],
16
- allScrapedLinks: [...state.allScrapedLinks],
17
- aspects: { ...state.aspects }
18
- };
19
-
20
- switch (event.type) {
21
- case 'PLANNING_COMPLETE': {
22
- newState.initialAgenda = event.agenda;
23
- newState.status = 'researching';
24
- // Assign first batch to aspects
25
- event.agenda.slice(0, event.initialCount).forEach((q: string, i: number) => {
26
- const id = `1.${i + 1}`;
27
- newState.aspects[id] = { id, query: q, status: 'pending' };
28
- });
29
- break;
30
- }
31
-
32
- case 'SIBLING_STARTED': {
33
- const aspect = newState.aspects[event.id];
34
- if (aspect) {
35
- newState.aspects[event.id] = {
36
- ...aspect,
37
- status: 'running'
38
- };
39
- }
40
- break;
41
- }
42
-
43
- case 'SIBLING_COMPLETED': {
44
- const aspect = newState.aspects[event.id];
45
- if (aspect) {
46
- newState.aspects[event.id] = {
47
- ...aspect,
48
- status: 'completed',
49
- report: event.report
50
- };
51
- }
52
- break;
53
- }
54
-
55
- case 'SIBLING_FAILED': {
56
- const aspect = newState.aspects[event.id];
57
- if (aspect) {
58
- newState.aspects[event.id] = {
59
- ...aspect,
60
- status: 'failed',
61
- error: event.error
62
- };
63
- }
64
- break;
65
- }
66
-
67
- case 'SIBLING_TOKENS': {
68
- const aspect = newState.aspects[event.id];
69
- if (aspect) {
70
- // Accumulate tokens and cost for this sibling
71
- const currentTokens = aspect.tokens || 0;
72
- const currentCost = aspect.cost || 0;
73
- newState.aspects[event.id] = {
74
- ...aspect,
75
- tokens: currentTokens + event.tokens,
76
- cost: currentCost + event.cost
77
- };
78
- }
79
- break;
80
- }
81
-
82
- case 'LINKS_SCRAPED': {
83
- newState.allScrapedLinks = [...new Set([...newState.allScrapedLinks, ...event.links])];
84
- break;
85
- }
86
-
87
- case 'PROMOTION_STARTED': {
88
- newState.promotedId = event.id;
89
- break;
90
- }
91
-
92
- case 'PROMOTION_DECISION': {
93
- if (event.finalSynthesis !== undefined) {
94
- // Case 1: Lead Evaluator chose to synthesize - complete with synthesis
95
- newState.finalSynthesis = event.finalSynthesis;
96
- newState.status = 'completed';
97
- newState.promotedId = undefined;
98
- } else if (event.nextQueries.length > 0 && newState.currentRound < event.maxRounds) {
99
- // Case 2: Lead Evaluator chose to continue AND we're under max rounds
100
- newState.currentRound++;
101
- newState.status = 'researching';
102
- newState.promotedId = undefined; // Reset for new round
103
- event.nextQueries.forEach((q: string, i: number) => {
104
- const id = `${newState.currentRound}.${i + 1}`;
105
- newState.aspects[id] = { id, query: q, status: 'pending' };
106
- });
107
- } else {
108
- // Case 3: Max rounds reached OR no next queries
109
- // Note: Per lead-evaluator prompt, when max rounds are reached, the evaluator
110
- // should synthesize. If nextQueries are provided at max rounds, this is a
111
- // protocol violation, and we enforce the constraint by completing anyway.
112
- newState.status = 'completed';
113
- newState.promotedId = undefined;
114
- }
115
- break;
116
- }
117
- }
118
-
119
- return newState;
120
- }
121
-
122
- /**
123
- * Utility to check if a round is finished
124
- */
125
- export function isRoundComplete(state: SystemResearchState, round: number): boolean {
126
- const roundAspects = Object.values(state.aspects).filter(a => a.id.startsWith(`${round}.`));
127
- if (roundAspects.length === 0) return false;
128
- return roundAspects.every(a => a.status === 'completed' || a.status === 'failed');
129
- }
@@ -1,63 +0,0 @@
1
- /**
2
- * Researcher ID Utilities
3
- *
4
- * Abstracts hierarchical IDs (1.1, 1.2, 2.1) to sequential display numbers (1, 2, 3)
5
- * for cleaner UI while maintaining internal structure.
6
- */
7
-
8
- import type { SystemResearchState } from './deep-research-types.ts';
9
-
10
- /**
11
- * Get sequential display number for a researcher ID
12
- *
13
- * Maps internal hierarchical IDs to sequential display numbers:
14
- * - Round 1: 1.1 → 1, 1.2 → 2, 1.3 → 3
15
- * - Round 2: 2.1 → 4, 2.2 → 5, 2.3 → 6
16
- * - Continue sequentially across all rounds
17
- */
18
- export function getDisplayNumber(state: SystemResearchState, internalId: string): string {
19
- const aspects = Object.values(state.aspects).sort((a, b) => {
20
- const aParts = a.id.split('.');
21
- const bParts = b.id.split('.');
22
- const aRound = parseInt(aParts[0] ?? '0');
23
- const aNum = parseInt(aParts[1] ?? '0');
24
- const bRound = parseInt(bParts[0] ?? '0');
25
- const bNum = parseInt(bParts[1] ?? '0');
26
-
27
- if (aRound !== bRound) return aRound - bRound;
28
- return aNum - bNum;
29
- });
30
-
31
- const index = aspects.findIndex(a => a.id === internalId);
32
- return index >= 0 ? (index + 1).toString() : internalId;
33
- }
34
-
35
- /**
36
- * Get a researcher's role context
37
- *
38
- * Returns role information for a researcher:
39
- * - Number in this round (1, 2, 3)
40
- * - Total siblings in this round
41
- * - Current round
42
- */
43
- export function getResearcherRoleContext(
44
- state: SystemResearchState,
45
- internalId: string
46
- ): {
47
- displayNumber: string;
48
- roundNumber: number;
49
- totalInRound: number;
50
- isLastInRound: boolean;
51
- } {
52
- const parts = internalId.split('.');
53
- const round = parseInt(parts[0] ?? '0');
54
- const sibling = parseInt(parts[1] ?? '0');
55
- const roundAspects = Object.values(state.aspects).filter(a => a.id.startsWith(`${round}.`));
56
-
57
- return {
58
- displayNumber: getDisplayNumber(state, internalId),
59
- roundNumber: round,
60
- totalInRound: roundAspects.length,
61
- isLastInRound: sibling === roundAspects.length,
62
- };
63
- }