@lincoln504/pi-research 0.1.12 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +64 -115
  3. package/dist/cli.mjs +23211 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +102 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +213 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +113 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +64 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1044 -0
  30. package/src/config.ts +1036 -147
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +264 -7
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -174
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -577
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -12,102 +12,27 @@
12
12
  */
13
13
 
14
14
  import type { Advisory, GitHubResult } from './types.ts';
15
+ import type { GitHubAdvisoryRaw } from './github-advisory-types.ts';
16
+ import { normalizeEcosystem } from './ecosystem.ts';
17
+ import { createTimeoutSignal, retryWithBackoff, isTransientError } from '../web-research/retry-utils.ts';
18
+ import { CircuitBreaker } from '../utils/circuit-breaker.ts';
19
+ import { metrics } from '../utils/metrics.ts';
15
20
  import { logger } from '../logger.ts';
16
- import { createTimeoutSignal, retryWithBackoff } from '../web-research/retry-utils.ts';
17
-
18
- // ============================================================================
19
- // Type Definitions for Fetch API
20
- // ============================================================================
21
-
22
- /**
23
- * Standard fetch API types for compatibility
24
- */
25
- type FetchType = (
26
- _input: string | Request,
27
- _init?: RequestInit,
28
- ) => Promise<Response>;
29
-
30
- declare const fetch: FetchType;
31
-
32
- interface Headers {
33
- append(_name: string, _value: string): void;
34
- delete(_name: string): void;
35
- get(_name: string): string | null;
36
- has(_name: string): boolean;
37
- set(_name: string, _value: string): void;
38
- forEach(_callback: (_value: string, _name: string) => void): void;
39
- }
40
-
41
- interface Response {
42
- readonly headers: Headers;
43
- readonly ok: boolean;
44
- readonly redirected: boolean;
45
- readonly status: number;
46
- readonly statusText: string;
47
- readonly type: 'basic' | 'cors' | 'default' | 'error' | 'opaque' | 'opaqueredirect';
48
- readonly url: string;
49
- readonly body: ReadableStream<Uint8Array> | null;
50
- readonly bodyUsed: boolean;
51
- clone(): Response;
52
- arrayBuffer(): Promise<ArrayBuffer>;
53
- blob(): Promise<Blob>;
54
- formData(): Promise<FormData>;
55
- json(): Promise<unknown>;
56
- text(): Promise<string>;
57
- }
58
-
59
- // ============================================================================
60
- // Type Definitions for GitHub API Responses
61
- // ============================================================================
62
-
63
- /**
64
- * GitHub Advisory Vulnerability Package
65
- */
66
- interface GitHubAdvisoryPackage {
67
- readonly ecosystem?: string;
68
- readonly name?: string;
69
- }
70
-
71
- /**
72
- * GitHub Advisory Affected Entry
73
- */
74
- interface GitHubAdvisoryAffected {
75
- readonly package?: GitHubAdvisoryPackage;
76
- }
77
-
78
- /**
79
- * GitHub Advisory Vulnerability
80
- */
81
- interface GitHubAdvisoryVulnerability {
82
- readonly package?: GitHubAdvisoryPackage;
83
- readonly affected?: readonly GitHubAdvisoryAffected[];
84
- }
85
-
86
- /**
87
- * Raw GitHub Advisory from API response
88
- */
89
- interface GitHubAdvisoryRaw {
90
- readonly ghsa_id?: string;
91
- readonly id?: string;
92
- readonly summary?: string;
93
- readonly description?: string;
94
- readonly severity?: string;
95
- readonly published_at?: string;
96
- readonly updated_at?: string;
97
- readonly cve_id?: string;
98
- readonly html_url?: string;
99
- readonly advisory_url?: string;
100
- readonly vulnerabilities?: readonly GitHubAdvisoryVulnerability[];
101
- readonly affected?: readonly GitHubAdvisoryAffected[];
102
- }
103
-
104
- /**
105
- * GitHub API list response wrapper
106
- */
107
- interface GitHubAdvisoryListResponse {
108
- readonly items?: readonly GitHubAdvisoryRaw[];
109
- }
110
-
21
+ import {
22
+ isGitHubAdvisoryRaw,
23
+ isArray,
24
+ isGitHubAdvisoryListResponse,
25
+ isGitHubAdvisoryArray,
26
+ isSingleGitHubAdvisory,
27
+ mapGitHubAdvisory,
28
+ } from './github-advisory-types.ts';
29
+
30
+ const githubCircuitBreaker = new CircuitBreaker({
31
+ failureThreshold: 5,
32
+ resetTimeoutMs: 10000,
33
+ name: 'GitHub API',
34
+ isTransientError: isTransientError
35
+ });
111
36
 
112
37
  // ============================================================================
113
38
  // Constants
@@ -116,197 +41,18 @@ interface GitHubAdvisoryListResponse {
116
41
  const GITHUB_API_BASE = 'https://api.github.com';
117
42
  const DEFAULT_MAX_RESULTS = 20;
118
43
 
119
- // ============================================================================
120
- // Type Guards
121
- // ============================================================================
122
-
123
- /**
124
- * Type guard to check if value is a GitHubAdvisoryRaw
125
- */
126
- function isGitHubAdvisoryRaw(value: unknown): value is GitHubAdvisoryRaw {
127
- if (typeof value !== 'object' || value === null) {
128
- return false;
129
- }
130
- return true;
131
- }
132
-
133
- /**
134
- * Type guard to check if value is an array
135
- */
136
- function isArray(value: unknown): value is readonly unknown[] {
137
- return Array.isArray(value);
138
- }
139
-
140
- /**
141
- * Type guard to check if value is a string
142
- */
143
- function isString(value: unknown): value is string {
144
- return typeof value === 'string';
145
- }
146
-
147
- /**
148
- * Type guard to check if value is an object (non-null, non-array)
149
- */
150
- function isObject(value: unknown): value is Record<string, unknown> {
151
- return typeof value === 'object' && value !== null && !isArray(value);
152
- }
153
-
154
- /**
155
- * Type guard for GitHubAdvisoryPackage
156
- */
157
- function isGitHubAdvisoryPackage(value: unknown): value is GitHubAdvisoryPackage {
158
- if (!isObject(value)) {
159
- return false;
160
- }
161
- return true;
162
- }
163
-
164
- /**
165
- * Type guard for GitHubAdvisoryAffected
166
- */
167
- function isGitHubAdvisoryAffected(value: unknown): value is GitHubAdvisoryAffected {
168
- if (!isObject(value)) {
169
- return false;
170
- }
171
- return true;
172
- }
173
-
174
- /**
175
- * Type guard for GitHubAdvisoryVulnerability
176
- */
177
- function isGitHubAdvisoryVulnerability(value: unknown): value is GitHubAdvisoryVulnerability {
178
- if (!isObject(value)) {
179
- return false;
180
- }
181
- return true;
182
- }
183
-
184
- /**
185
- * Type guard to check if response is a list response with items
186
- */
187
- function isGitHubAdvisoryListResponse(value: unknown): value is GitHubAdvisoryListResponse {
188
- if (!isObject(value)) {
189
- return false;
190
- }
191
- return 'items' in value;
192
- }
193
-
194
- /**
195
- * Type guard to check if response is an array of advisory objects
196
- */
197
- function isGitHubAdvisoryArray(value: unknown): value is GitHubAdvisoryRaw[] {
198
- if (!isArray(value)) {
199
- return false;
200
- }
201
- return value.every(isGitHubAdvisoryRaw);
202
- }
203
-
204
- /**
205
- * Type guard to check if response is a single advisory object
206
- */
207
- function isSingleGitHubAdvisory(value: unknown): value is GitHubAdvisoryRaw {
208
- if (!isGitHubAdvisoryRaw(value)) {
209
- return false;
210
- }
211
- // Single advisory typically has ghsa_id
212
- return 'ghsa_id' in value || 'summary' in value;
213
- }
214
-
215
- // ============================================================================
216
- // Helper Functions
217
- // ============================================================================
218
-
219
44
  /**
220
- * Extract affected packages from an advisory item
45
+ * Build GitHub API request headers, attaching a bearer token when GITHUB_TOKEN
46
+ * is set. Authentication raises the rate limit from 60/hr to 5000/hr — without
47
+ * it the unauthenticated ceiling is hit almost immediately under real use.
221
48
  */
222
- function extractAffectedPackages(item: GitHubAdvisoryRaw): string[] {
223
- const affectedPackages: string[] = [];
224
-
225
- // Try different possible structures for affected packages
226
- // GitHub API response structure may vary between endpoints
227
-
228
- // Check for vulnerabilities array
229
- if (item.vulnerabilities !== undefined && isArray(item.vulnerabilities)) {
230
- for (const vuln of item.vulnerabilities) {
231
- if (!isGitHubAdvisoryVulnerability(vuln)) {
232
- continue;
233
- }
234
-
235
- // Check for package in various possible locations
236
- if (vuln.package !== undefined) {
237
- if (isGitHubAdvisoryPackage(vuln.package)) {
238
- const ecosystem = vuln.package.ecosystem ?? '';
239
- const name = vuln.package.name;
240
-
241
- if (isString(name) && name !== '') {
242
- if (ecosystem !== '') {
243
- affectedPackages.push(`${ecosystem}/${name}`);
244
- } else {
245
- affectedPackages.push(name);
246
- }
247
- }
248
- }
249
- } else if (vuln.affected !== undefined && isArray(vuln.affected)) {
250
- // Alternative structure with 'affected' array
251
- for (const aff of vuln.affected) {
252
- if (isGitHubAdvisoryAffected(aff) && aff.package !== undefined) {
253
- const pkg = aff.package;
254
- if (isGitHubAdvisoryPackage(pkg)) {
255
- const ecosystem = pkg.ecosystem ?? '';
256
- const name = pkg.name;
257
-
258
- if (isString(name) && name !== '') {
259
- if (ecosystem !== '') {
260
- affectedPackages.push(`${ecosystem}/${name}`);
261
- } else {
262
- affectedPackages.push(name);
263
- }
264
- }
265
- }
266
- }
267
- }
268
- }
269
- }
270
- }
271
-
272
- // Check for direct 'affected' array at the top level
273
- if (affectedPackages.length === 0 && item.affected !== undefined && isArray(item.affected)) {
274
- for (const aff of item.affected) {
275
- if (isGitHubAdvisoryAffected(aff) && aff.package !== undefined) {
276
- const pkg = aff.package;
277
- if (isGitHubAdvisoryPackage(pkg)) {
278
- const ecosystem = pkg.ecosystem ?? '';
279
- const name = pkg.name;
280
-
281
- if (isString(name) && name !== '') {
282
- if (ecosystem !== '') {
283
- affectedPackages.push(`${ecosystem}/${name}`);
284
- } else {
285
- affectedPackages.push(name);
286
- }
287
- }
288
- }
289
- }
290
- }
291
- }
292
-
293
- return affectedPackages;
294
- }
295
-
296
- /**
297
- * Extract references from an advisory item
298
- */
299
- function extractReferences(item: GitHubAdvisoryRaw): string[] {
300
- const references: string[] = [];
301
-
302
- if (item.html_url !== undefined && item.html_url !== '') {
303
- references.push(item.html_url);
304
- }
305
- if (item.advisory_url !== undefined && item.advisory_url !== '') {
306
- references.push(item.advisory_url);
307
- }
308
-
309
- return references;
49
+ function githubHeaders(): Record<string, string> {
50
+ const token = process.env['GITHUB_TOKEN'];
51
+ return {
52
+ 'User-Agent': 'pi-research/2.0',
53
+ Accept: 'application/vnd.github.v3+json',
54
+ ...(token ? { Authorization: `Bearer ${token}` } : {}),
55
+ };
310
56
  }
311
57
 
312
58
  // ============================================================================
@@ -327,9 +73,18 @@ export async function searchGitHubAdvisories(
327
73
  readonly severity?: string; // LOW, MODERATE, HIGH, CRITICAL
328
74
  readonly maxResults?: number;
329
75
  readonly repo?: string; // "owner/repo" format
76
+ readonly signal?: AbortSignal;
330
77
  },
331
78
  ): Promise<GitHubResult> {
79
+ const startTime = Date.now();
332
80
  const maxResults = options?.maxResults ?? DEFAULT_MAX_RESULTS;
81
+ // GitHub caps per_page at 100 server-side and these paths do not follow the Link header,
82
+ // so a request for >100 silently returns ≤100 with no signal that more exist — a security
83
+ // tool must not imply completeness it didn't fetch. Clamp the page size and warn.
84
+ const perPage = Math.min(100, maxResults);
85
+ if (maxResults > 100) {
86
+ logger.warn(`[github-advisories] maxResults=${maxResults} exceeds GitHub's 100-per-page cap and these paths don't paginate; returning up to 100. Results may be incomplete.`);
87
+ }
333
88
  const advisories: Advisory[] = [];
334
89
  let error: string | undefined = undefined;
335
90
 
@@ -345,22 +100,27 @@ export async function searchGitHubAdvisories(
345
100
  }
346
101
 
347
102
  const [owner, name] = repoParts;
348
- const url = `${GITHUB_API_BASE}/repos/${owner}/${name}/security-advisories?per_page=${maxResults}`;
103
+ // Constrain to GitHub's allowed owner/repo character set before interpolating
104
+ // into the API path. encodeURIComponent already neutralises traversal, but this
105
+ // rejects malformed input early instead of issuing a doomed request.
106
+ const repoNameRe = /^[A-Za-z0-9._-]+$/;
107
+ if (!repoNameRe.test(owner!) || !repoNameRe.test(name!)) {
108
+ throw new Error(`Invalid repo format: "${options.repo}". Owner and name may only contain letters, digits, '.', '_' and '-'.`);
109
+ }
110
+ const url = `${GITHUB_API_BASE}/repos/${encodeURIComponent(owner!)}/${encodeURIComponent(name!)}/security-advisories?per_page=${perPage}`;
349
111
 
350
- const response = await retryWithBackoff(async (): Promise<Response> => {
112
+ const response = await githubCircuitBreaker.execute(() => retryWithBackoff(async () => {
351
113
  const resp = await fetch(url, {
352
- headers: {
353
- 'User-Agent': 'pi-research/2.0',
354
- 'Accept': 'application/vnd.github.v3+json',
355
- },
356
- signal: createTimeoutSignal(30000),
114
+ headers: githubHeaders(),
115
+ signal: createTimeoutSignal(10000, options?.signal),
357
116
  });
358
117
 
359
- if (resp.ok === false) {
118
+ if (!resp.ok) {
360
119
  if (resp.status === 404) {
361
120
  throw new Error(`Repository "${owner}/${name}" not found or no access to security advisories.`);
362
121
  }
363
122
  if (resp.status === 403) {
123
+ metrics.increment('github_ratelimit_hits_total', 1, { endpoint: 'repo_advisories' });
364
124
  throw new Error('GitHub API rate limit exceeded (HTTP 403). Retrying with backoff...');
365
125
  }
366
126
  if (resp.status >= 500) {
@@ -369,20 +129,37 @@ export async function searchGitHubAdvisories(
369
129
  throw new Error(`GitHub API error (${resp.status}): ${resp.statusText}`);
370
130
  }
371
131
 
132
+ // Track rate limit headers
133
+ const rateLimitRemaining = resp.headers?.get ? resp.headers.get('X-RateLimit-Remaining') : null;
134
+ const rateLimitLimit = resp.headers?.get ? resp.headers.get('X-RateLimit-Limit') : null;
135
+ if (rateLimitRemaining !== null) {
136
+ metrics.setGauge('github_ratelimit_remaining', parseInt(rateLimitRemaining, 10), { endpoint: 'repo_advisories' });
137
+ }
138
+ if (rateLimitLimit !== null) {
139
+ metrics.setGauge('github_ratelimit_limit', parseInt(rateLimitLimit, 10), { endpoint: 'repo_advisories' });
140
+ }
141
+
142
+ metrics.increment('github_requests_total', 1, { endpoint: 'repo_advisories', status: 'success' });
372
143
  return resp;
373
144
  }, {
374
145
  maxRetries: 2,
375
146
  initialDelay: 1000,
376
147
  maxDelay: 5000,
377
- });
378
-
379
- const data = await response.json();
148
+ signal: options?.signal,
149
+ }));
150
+
151
+ let data: unknown;
152
+ try {
153
+ data = await response.json();
154
+ } catch {
155
+ metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json' });
156
+ throw new Error('GitHub API returned a malformed JSON response (repo advisories).');
157
+ }
380
158
  let repoAdvisories: readonly GitHubAdvisoryRaw[] = [];
381
159
 
382
160
  if (isGitHubAdvisoryArray(data)) {
383
161
  repoAdvisories = data;
384
162
  } else if (isGitHubAdvisoryListResponse(data) && isArray(data.items)) {
385
- // Filter to ensure only valid advisory objects
386
163
  repoAdvisories = data.items.filter(isGitHubAdvisoryRaw);
387
164
  } else if (isSingleGitHubAdvisory(data)) {
388
165
  repoAdvisories = [data];
@@ -390,41 +167,50 @@ export async function searchGitHubAdvisories(
390
167
 
391
168
  allAdvisories = repoAdvisories.map(mapGitHubAdvisory);
392
169
  } else {
393
- // Search global advisories.
394
- // The GitHub advisories API supports cve_id and ghsa_id as query params,
395
- // so use those for precise lookups; fall back to a general listing for
396
- // keyword terms (client-side filtering is applied afterwards).
397
- const termResults: Advisory[] = [];
398
-
399
- for (const term of terms) {
170
+ // Search global advisories. Each term is fetched independently through the
171
+ // circuit breaker and settled in isolation (Promise.allSettled) so one bad
172
+ // term — a transient 5xx, a malformed body, an exhausted retry — never
173
+ // discards the advisories already gathered for the others.
174
+ const fetchTerm = async (term: string): Promise<Advisory[]> => {
400
175
  const termUpper = term.toUpperCase();
401
176
  let apiUrl: string;
177
+ let endpointType: string;
402
178
 
403
179
  if (termUpper.startsWith('CVE-')) {
404
- // Use cve_id param for CVE lookups
405
- apiUrl = `${GITHUB_API_BASE}/advisories?cve_id=${encodeURIComponent(termUpper)}&per_page=${maxResults}`;
180
+ apiUrl = `${GITHUB_API_BASE}/advisories?cve_id=${encodeURIComponent(termUpper)}&per_page=${perPage}`;
181
+ endpointType = 'cve_lookup';
406
182
  } else if (termUpper.startsWith('GHSA-')) {
407
- // Direct advisory lookup by GHSA ID
408
183
  apiUrl = `${GITHUB_API_BASE}/advisories/${encodeURIComponent(term)}`;
184
+ endpointType = 'ghsa_lookup';
409
185
  } else {
410
- // Keyword: fetch recent published advisories and filter client-side
411
- apiUrl = `${GITHUB_API_BASE}/advisories?per_page=${maxResults}&state=published&direction=desc`;
186
+ // Server-side narrowing so the term can match beyond just the newest
187
+ // `maxResults` global advisories. GitHub's /advisories endpoint has no
188
+ // free-text search, but it does support `affects=<package>`; a single-token
189
+ // term (npm/pip/maven package identifiers never contain whitespace, incl.
190
+ // scoped `@scope/name` and `group:artifact`) is treated as a package name.
191
+ // Multi-word terms keep the newest-N + client-side substring path, since
192
+ // `affects` on a non-package phrase would just return nothing.
193
+ const ecosystemParam =
194
+ options?.ecosystem !== undefined && options.ecosystem !== ''
195
+ ? `&ecosystem=${encodeURIComponent(normalizeEcosystem(options.ecosystem, 'github'))}`
196
+ : '';
197
+ const affectsParam = !/\s/.test(term) ? `&affects=${encodeURIComponent(term)}` : '';
198
+ apiUrl = `${GITHUB_API_BASE}/advisories?per_page=${perPage}&state=published&direction=desc${ecosystemParam}${affectsParam}`;
199
+ endpointType = 'search';
412
200
  }
413
201
 
414
- const response = await retryWithBackoff(async (): Promise<Response> => {
202
+ const response = await githubCircuitBreaker.execute(() => retryWithBackoff(async () => {
415
203
  const resp = await fetch(apiUrl, {
416
- headers: {
417
- 'User-Agent': 'pi-research/2.0',
418
- 'Accept': 'application/vnd.github.v3+json',
419
- },
420
- signal: createTimeoutSignal(30000),
204
+ headers: githubHeaders(),
205
+ signal: createTimeoutSignal(10000, options?.signal),
421
206
  });
422
207
 
423
- if (resp.ok === false) {
208
+ if (!resp.ok) {
424
209
  if (resp.status === 404) {
425
210
  throw new Error('Advisory not found (HTTP 404)');
426
211
  }
427
212
  if (resp.status === 403) {
213
+ metrics.increment('github_ratelimit_hits_total', 1, { endpoint: endpointType });
428
214
  throw new Error('GitHub API rate limit exceeded (HTTP 403). Retrying with backoff...');
429
215
  }
430
216
  if (resp.status >= 500) {
@@ -433,17 +219,33 @@ export async function searchGitHubAdvisories(
433
219
  throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
434
220
  }
435
221
 
222
+ const rateLimitRemaining = resp.headers?.get ? resp.headers.get('X-RateLimit-Remaining') : null;
223
+ const rateLimitLimit = resp.headers?.get ? resp.headers.get('X-RateLimit-Limit') : null;
224
+ if (rateLimitRemaining !== null) {
225
+ metrics.setGauge('github_ratelimit_remaining', parseInt(rateLimitRemaining, 10), { endpoint: endpointType });
226
+ }
227
+ if (rateLimitLimit !== null) {
228
+ metrics.setGauge('github_ratelimit_limit', parseInt(rateLimitLimit, 10), { endpoint: endpointType });
229
+ }
230
+ metrics.increment('github_requests_total', 1, { endpoint: endpointType, status: 'success' });
436
231
  return resp;
437
232
  }, {
438
233
  maxRetries: 2,
439
234
  initialDelay: 1000,
440
235
  maxDelay: 5000,
441
- });
442
-
443
- const data = await response.json();
236
+ signal: options?.signal,
237
+ }));
238
+
239
+ // A 404 on a direct GHSA/CVE lookup means "no such advisory", not failure.
240
+ let data: unknown;
241
+ try {
242
+ data = await response.json();
243
+ } catch {
244
+ metrics.increment('github_search_errors_total', 1, { error_type: 'malformed_json', endpoint: endpointType });
245
+ throw new Error(`GitHub API returned a malformed JSON response for term "${term}".`);
246
+ }
444
247
  let items: readonly GitHubAdvisoryRaw[] = [];
445
248
 
446
- // GHSA direct lookup returns an object; list endpoints return arrays
447
249
  if (isGitHubAdvisoryArray(data)) {
448
250
  items = data;
449
251
  } else if (isSingleGitHubAdvisory(data)) {
@@ -452,45 +254,72 @@ export async function searchGitHubAdvisories(
452
254
  items = data.items.filter(isGitHubAdvisoryRaw);
453
255
  }
454
256
 
455
- termResults.push(...items.map(mapGitHubAdvisory));
257
+ metrics.increment(items.length > 0 ? 'github_cache_hits_total' : 'github_cache_misses_total', 1, { term, endpoint: endpointType });
258
+
259
+ return items.map(mapGitHubAdvisory);
260
+ };
261
+
262
+ const settled = await Promise.allSettled(terms.map(fetchTerm));
263
+ const termResults: Advisory[] = [];
264
+ const failures: string[] = [];
265
+ settled.forEach((result, i) => {
266
+ if (result.status === 'fulfilled') {
267
+ termResults.push(...result.value);
268
+ } else {
269
+ const reason = result.reason instanceof Error ? result.reason.message : String(result.reason);
270
+ // A direct-lookup 404 is an expected "not found", not a real failure.
271
+ if (!reason.includes('HTTP 404')) failures.push(`${terms[i]} (${reason})`);
272
+ }
273
+ });
274
+ if (failures.length > 0 && failures.length === terms.length) {
275
+ // Every term failed — surface it; partial failures are tolerated silently.
276
+ throw new Error(`All GitHub advisory lookups failed: ${failures.join('; ')}`);
277
+ } else if (failures.length > 0) {
278
+ error = `Some GitHub advisory lookups failed: ${failures.join('; ')}`;
456
279
  }
457
280
 
458
281
  // Deduplicate by GHSA ID
459
282
  const seen = new Set<string>();
460
283
  for (const adv of termResults) {
461
- if (!seen.has(adv.id)) {
284
+ if (adv.id && !seen.has(adv.id)) {
462
285
  seen.add(adv.id);
463
286
  allAdvisories.push(adv);
287
+ } else if (!adv.id) {
288
+ allAdvisories.push(adv);
464
289
  }
465
290
  }
466
291
  }
467
292
 
468
293
  // Filter by search terms if provided (OR logic: match any term)
469
294
  if (terms.length > 0) {
470
- allAdvisories = allAdvisories.filter((adv): boolean =>
471
- terms.some((term): boolean => {
295
+ allAdvisories = allAdvisories.filter((adv): boolean => {
296
+ if (terms.length === 0) return true;
297
+
298
+ return terms.some((term): boolean => {
472
299
  const t = term.toLowerCase();
473
- const advId = adv.id.toLowerCase();
474
- const advSummary = adv.summary.toLowerCase();
475
- const advDescription = adv.description !== undefined
476
- ? adv.description.toLowerCase()
477
- : '';
478
- const advCveId = adv.cveId !== undefined
479
- ? adv.cveId.toLowerCase()
480
- : '';
300
+ const advId = (adv.id || '').toLowerCase();
301
+ const advSummary = (adv.summary || '').toLowerCase();
302
+ const advDescription = (adv.description || '').toLowerCase();
303
+ const advCveId = (adv.cveId || '').toLowerCase();
481
304
 
305
+ // Precise match for IDs
306
+ if (advId === t || advCveId === t) return true;
307
+
308
+ // Partial match for everything else. Include affectedPackages so a
309
+ // server-side `affects=<package>` hit is not then dropped just because
310
+ // the advisory summary/description doesn't repeat the package name.
482
311
  return (
483
312
  advId.includes(t) ||
484
313
  advCveId.includes(t) ||
485
314
  advSummary.includes(t) ||
486
- advDescription.includes(t)
315
+ advDescription.includes(t) ||
316
+ adv.affectedPackages.some(p => p.toLowerCase().includes(t))
487
317
  );
488
- }),
489
- );
318
+ });
319
+ });
490
320
  }
491
321
 
492
322
  // Filter by severity if provided
493
- // GitHub uses "MODERATE" where the rest of the industry uses "MEDIUM"
494
323
  if (options?.severity !== undefined && options.severity !== '') {
495
324
  const severity = options.severity.toUpperCase();
496
325
  const githubSeverity = severity === 'MEDIUM' ? 'MODERATE' : severity;
@@ -503,6 +332,10 @@ export async function searchGitHubAdvisories(
503
332
 
504
333
  } catch (err: unknown) {
505
334
  error = err instanceof Error ? err.message : String(err);
335
+ metrics.increment('github_search_errors_total', 1, { error_type: err instanceof Error ? err.name : 'unknown' });
336
+ } finally {
337
+ const duration = Date.now() - startTime;
338
+ metrics.observe('github_search_duration_ms', duration, { has_error: error ? 'true' : 'false' });
506
339
  }
507
340
 
508
341
  return {
@@ -512,96 +345,3 @@ export async function searchGitHubAdvisories(
512
345
  };
513
346
  }
514
347
 
515
- /**
516
- * Get specific advisory by GHSA ID or CVE ID
517
- *
518
- * @param id - The GHSA ID or CVE ID to fetch
519
- * @returns Promise resolving to Advisory or null if not found
520
- */
521
- export async function getAdvisoryById(id: string): Promise<Advisory | null> {
522
- try {
523
- if (id === '') {
524
- return null;
525
- }
526
-
527
- const url = `${GITHUB_API_BASE}/advisories/${encodeURIComponent(id)}`;
528
-
529
- const response = await retryWithBackoff(async (): Promise<Response> => {
530
- const resp = await fetch(url, {
531
- headers: {
532
- 'User-Agent': 'pi-research/2.0',
533
- 'Accept': 'application/vnd.github.v3+json',
534
- },
535
- signal: createTimeoutSignal(30000),
536
- });
537
-
538
- if (resp.ok === false) {
539
- if (resp.status === 404) {
540
- throw new Error('Advisory not found (HTTP 404)');
541
- }
542
- if (resp.status === 403) {
543
- throw new Error('GitHub API rate limit exceeded (HTTP 403). Retrying with backoff...');
544
- }
545
- if (resp.status >= 500) {
546
- throw new Error(`GitHub server error (HTTP ${resp.status}). Retrying with backoff...`);
547
- }
548
- throw new Error(`HTTP ${resp.status}: ${resp.statusText}`);
549
- }
550
-
551
- return resp;
552
- }, {
553
- maxRetries: 2,
554
- initialDelay: 1000,
555
- maxDelay: 5000,
556
- });
557
-
558
- const data = await response.json();
559
-
560
- if (isGitHubAdvisoryRaw(data)) {
561
- return mapGitHubAdvisory(data);
562
- }
563
-
564
- return null;
565
- } catch (err: unknown) {
566
- // Check if it's a 404 error (not found) - return null instead of throwing
567
- if (err instanceof Error && err.message.includes('HTTP 404')) {
568
- logger.warn(`[GitHub Advisories] Advisory ${id} not found`);
569
- return null;
570
- }
571
- logger.error(`[GitHub Advisories] Error fetching advisory ${id}:`, err);
572
- return null;
573
- }
574
- }
575
-
576
- /**
577
- * Map GitHub advisory to our Advisory interface
578
- *
579
- * @param item - Raw GitHub advisory object
580
- * @returns Mapped Advisory object
581
- */
582
- function mapGitHubAdvisory(item: GitHubAdvisoryRaw): Advisory {
583
- const ghsaId = item.ghsa_id ?? item.id ?? '';
584
- const summary = item.summary ?? '';
585
- const description = item.description ?? '';
586
- const severityRaw = item.severity ?? 'UNKNOWN';
587
- const severity = severityRaw.toUpperCase();
588
- const published = item.published_at ?? '';
589
- const modified = item.updated_at ?? '';
590
- const cveId = item.cve_id ?? '';
591
-
592
- const affectedPackages = extractAffectedPackages(item);
593
- const references = extractReferences(item);
594
-
595
- return {
596
- id: ghsaId,
597
- source: 'github',
598
- severity,
599
- summary,
600
- description,
601
- published,
602
- modified,
603
- cveId,
604
- references,
605
- affectedPackages,
606
- };
607
- }