@lincoln504/pi-research 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +131 -0
  3. package/config/searxng/default-settings.yml +189 -0
  4. package/config/searxng/limiter.toml +34 -0
  5. package/config/tooling/eslint.config.js +73 -0
  6. package/config/tooling/vitest.config.integration.ts +17 -0
  7. package/config/tooling/vitest.config.ts +40 -0
  8. package/config/tooling/vitest.config.unit.ts +12 -0
  9. package/docs/ARCHITECTURE.md +109 -0
  10. package/docs/assets/README-banner.jpg +0 -0
  11. package/package.json +92 -0
  12. package/scripts/setup.js +174 -0
  13. package/src/config.ts +206 -0
  14. package/src/constants.ts +148 -0
  15. package/src/healthcheck/index.ts +309 -0
  16. package/src/index.ts +71 -0
  17. package/src/infrastructure/searxng-lifecycle.ts +577 -0
  18. package/src/infrastructure/searxng-manager.ts +1173 -0
  19. package/src/infrastructure/state-manager.ts +918 -0
  20. package/src/logger.ts +246 -0
  21. package/src/orchestration/deep-research-orchestrator.ts +848 -0
  22. package/src/orchestration/deep-research-reducer.ts +129 -0
  23. package/src/orchestration/deep-research-types.ts +54 -0
  24. package/src/orchestration/id-utils.ts +63 -0
  25. package/src/orchestration/researcher.ts +104 -0
  26. package/src/orchestration/session-context.ts +43 -0
  27. package/src/prompts/research-tool-usage.md +45 -0
  28. package/src/prompts/researcher.md +166 -0
  29. package/src/prompts/system-coordinator.md +24 -0
  30. package/src/prompts/system-lead-evaluator.md +225 -0
  31. package/src/security/cisa-kev.ts +242 -0
  32. package/src/security/github-advisories.ts +607 -0
  33. package/src/security/index.ts +406 -0
  34. package/src/security/nvd.ts +532 -0
  35. package/src/security/osv.ts +487 -0
  36. package/src/security/types.ts +138 -0
  37. package/src/stackexchange/cache.ts +111 -0
  38. package/src/stackexchange/index.ts +352 -0
  39. package/src/stackexchange/output/compact.ts +64 -0
  40. package/src/stackexchange/output/index.ts +7 -0
  41. package/src/stackexchange/output/json.ts +21 -0
  42. package/src/stackexchange/output/table.ts +123 -0
  43. package/src/stackexchange/queries.ts +168 -0
  44. package/src/stackexchange/rest-client.ts +127 -0
  45. package/src/stackexchange/types.ts +129 -0
  46. package/src/tool.ts +430 -0
  47. package/src/tools/grep.ts +335 -0
  48. package/src/tools/index.ts +77 -0
  49. package/src/tools/scrape.ts +451 -0
  50. package/src/tools/search.ts +188 -0
  51. package/src/tools/security.ts +292 -0
  52. package/src/tools/stackexchange.ts +101 -0
  53. package/src/tui/research-panel.ts +576 -0
  54. package/src/tui/searxng-status.ts +80 -0
  55. package/src/types/extension-context.ts +60 -0
  56. package/src/types/llm.ts +111 -0
  57. package/src/utils/inject-date.ts +29 -0
  58. package/src/utils/input-validation.ts +103 -0
  59. package/src/utils/json-utils.ts +262 -0
  60. package/src/utils/make-resource-loader.ts +43 -0
  61. package/src/utils/research-error.ts +205 -0
  62. package/src/utils/research-export.ts +199 -0
  63. package/src/utils/searxng-config.ts +94 -0
  64. package/src/utils/session-state.ts +295 -0
  65. package/src/utils/shared-links.ts +123 -0
  66. package/src/utils/shutdown-manager.ts +54 -0
  67. package/src/utils/text-utils.ts +83 -0
  68. package/src/utils/tool-usage-tracker.ts +135 -0
  69. package/src/web-research/dependencies.ts +20 -0
  70. package/src/web-research/retry-utils.ts +232 -0
  71. package/src/web-research/scrapers.ts +935 -0
  72. package/src/web-research/search.ts +235 -0
  73. package/src/web-research/types.ts +47 -0
  74. package/src/web-research/utils.ts +158 -0
  75. package/tsconfig.json +40 -0
@@ -0,0 +1,406 @@
1
+ /**
2
+ * Security Databases Module
3
+ *
4
+ * Main entry point for security database integrations
5
+ * Combines multiple vulnerability database sources
6
+ * Refactored with dependency injection for testability
7
+ */
8
+
9
+ import type {
10
+ SecuritySearchParams,
11
+ SecuritySearchResult,
12
+ INVDClient,
13
+ ICisaKevClient,
14
+ IGitHubAdvisoriesClient,
15
+ IOSVClient,
16
+ NVDSearchOptions,
17
+ CisaKevSearchOptions,
18
+ GitHubSearchOptions,
19
+ OSVSearchOptions,
20
+ NVDResult,
21
+ CisaKevResult,
22
+ GitHubResult,
23
+ OSVResult,
24
+ } from './types.ts';
25
+ import { REQUEST_DELAY_MS } from '../constants.ts';
26
+ import { searchNVD } from './nvd.ts';
27
+ import { searchCisaKev } from './cisa-kev.ts';
28
+ import { searchGitHubAdvisories } from './github-advisories.ts';
29
+ import { searchOSV } from './osv.ts';
30
+
31
+ // ============================================================================
32
+ // Type Definitions
33
+ // ============================================================================
34
+
35
+ type Severity = 'LOW' | 'MEDIUM' | 'HIGH' | 'CRITICAL';
36
+
37
+ interface DatabaseInfo {
38
+ readonly databases: readonly DatabaseInfoEntry[];
39
+ }
40
+
41
+ interface DatabaseInfoEntry {
42
+ readonly id: string;
43
+ readonly name: string;
44
+ readonly description: string;
45
+ readonly url: string;
46
+ readonly free: boolean;
47
+ }
48
+
49
+ // ============================================================================
50
+ // Helper Functions
51
+ // ============================================================================
52
+
53
+ /**
54
+ * Type guard to check if a value is a valid Severity
55
+ */
56
+ function isValidSeverity(value: unknown): value is Severity {
57
+ return (
58
+ typeof value === 'string' &&
59
+ (value === 'LOW' || value === 'MEDIUM' || value === 'HIGH' || value === 'CRITICAL')
60
+ );
61
+ }
62
+
63
+ /**
64
+ * Safely extract severity from params, returning undefined if invalid
65
+ */
66
+ function getSeverityParam(params: SecuritySearchParams): Severity | undefined {
67
+ if (params.severity === undefined) {
68
+ return undefined;
69
+ }
70
+ return isValidSeverity(params.severity) ? params.severity : undefined;
71
+ }
72
+
73
+ /**
74
+ * Safely extract error message from unknown error
75
+ */
76
+ function getErrorMessage(error: unknown): string {
77
+ if (error instanceof Error) {
78
+ return error.message;
79
+ }
80
+ return String(error);
81
+ }
82
+
83
+ // ============================================================================
84
+ // Security Searcher Class (with dependency injection)
85
+ // ============================================================================
86
+
87
+ /**
88
+ * Configuration for SecuritySearcher
89
+ */
90
+ export interface SecuritySearcherConfig {
91
+ readonly nvdClient?: INVDClient;
92
+ readonly cisaKevClient?: ICisaKevClient;
93
+ readonly githubAdvisoriesClient?: IGitHubAdvisoriesClient;
94
+ readonly osvClient?: IOSVClient;
95
+ readonly requestDelay?: number; // milliseconds between requests
96
+ }
97
+
98
+ const DEFAULT_CONFIG: Required<SecuritySearcherConfig> = {
99
+ nvdClient: null as unknown as INVDClient,
100
+ cisaKevClient: null as unknown as ICisaKevClient,
101
+ githubAdvisoriesClient: null as unknown as IGitHubAdvisoriesClient,
102
+ osvClient: null as unknown as IOSVClient,
103
+ requestDelay: REQUEST_DELAY_MS,
104
+ };
105
+
106
+ /**
107
+ * SecuritySearcher - orchestrates searches across multiple security databases
108
+ * Uses dependency injection for testability
109
+ */
110
+ export class SecuritySearcher {
111
+ private readonly config: Required<SecuritySearcherConfig>;
112
+
113
+ constructor(config: Partial<SecuritySearcherConfig> = {}) {
114
+ this.config = { ...DEFAULT_CONFIG, ...config };
115
+ }
116
+
117
+ /**
118
+ * Main search function that queries multiple security databases
119
+ */
120
+ async search(params: SecuritySearchParams): Promise<SecuritySearchResult> {
121
+ const startTime = Date.now();
122
+ const results: { nvd?: NVDResult; cisa_kev?: CisaKevResult; github?: GitHubResult; osv?: OSVResult } = {};
123
+ const errors: string[] = [];
124
+ let totalVulnerabilities = 0;
125
+
126
+ const searchPromises: Promise<void>[] = [];
127
+
128
+ // Search each requested database in parallel
129
+ if (params.databases.includes('nvd')) {
130
+ searchPromises.push((async () => {
131
+ try {
132
+ const severity = getSeverityParam(params);
133
+ const nvdResult = await this.searchNVD(
134
+ params.terms,
135
+ {
136
+ severity: severity,
137
+ maxResults: params.maxResults,
138
+ includeExploited: params.includeExploited,
139
+ },
140
+ );
141
+ results.nvd = nvdResult;
142
+ totalVulnerabilities += nvdResult.count;
143
+ if (nvdResult.error !== undefined) {
144
+ errors.push(`NVD: ${nvdResult.error}`);
145
+ }
146
+ } catch (err: unknown) {
147
+ errors.push(`NVD: ${getErrorMessage(err)}`);
148
+ }
149
+ })());
150
+ }
151
+
152
+ if (params.databases.includes('cisa_kev')) {
153
+ searchPromises.push((async () => {
154
+ try {
155
+ const cisaResult = await this.searchCisaKev(params.terms, {
156
+ maxResults: params.maxResults,
157
+ });
158
+ results.cisa_kev = cisaResult;
159
+ totalVulnerabilities += cisaResult.count;
160
+ if (cisaResult.error !== undefined) {
161
+ errors.push(`CISA KEV: ${cisaResult.error}`);
162
+ }
163
+ } catch (err: unknown) {
164
+ errors.push(`CISA KEV: ${getErrorMessage(err)}`);
165
+ }
166
+ })());
167
+ }
168
+
169
+ if (params.databases.includes('github')) {
170
+ searchPromises.push((async () => {
171
+ try {
172
+ const githubResult = await this.searchGitHub(params.terms, {
173
+ ecosystem: params.ecosystem,
174
+ severity: params.severity,
175
+ maxResults: params.maxResults,
176
+ repo: params.githubRepo,
177
+ });
178
+ results.github = githubResult;
179
+ totalVulnerabilities += githubResult.count;
180
+ if (githubResult.error !== undefined) {
181
+ errors.push(`GitHub: ${githubResult.error}`);
182
+ }
183
+ } catch (err: unknown) {
184
+ errors.push(`GitHub: ${getErrorMessage(err)}`);
185
+ }
186
+ })());
187
+ }
188
+
189
+ if (params.databases.includes('osv')) {
190
+ searchPromises.push((async () => {
191
+ try {
192
+ const osvResult = await this.searchOSV(params.terms, {
193
+ ecosystem: params.ecosystem,
194
+ severity: params.severity,
195
+ maxResults: params.maxResults,
196
+ });
197
+ results.osv = osvResult;
198
+ totalVulnerabilities += osvResult.count;
199
+ if (osvResult.error !== undefined) {
200
+ errors.push(`OSV: ${osvResult.error}`);
201
+ }
202
+ } catch (err: unknown) {
203
+ errors.push(`OSV: ${getErrorMessage(err)}`);
204
+ }
205
+ })());
206
+ }
207
+
208
+ // Wait for all searches to complete
209
+ await Promise.all(searchPromises);
210
+
211
+ // Apply delay to avoid rate limiting for subsequent calls
212
+ if (this.config.requestDelay > 0) {
213
+ await new Promise<void>((resolve) => {
214
+ const timeoutId = setTimeout(resolve, this.config.requestDelay);
215
+ timeoutId.unref();
216
+ });
217
+ }
218
+
219
+ const duration = Date.now() - startTime;
220
+
221
+ return {
222
+ results,
223
+ totalDatabases: Object.keys(results).length,
224
+ totalVulnerabilities,
225
+ duration,
226
+ };
227
+ }
228
+
229
+ private async searchNVD(terms: readonly string[], options: NVDSearchOptions): Promise<import('./types.ts').NVDResult> {
230
+ const client = this.config.nvdClient ?? createDefaultNVDClient();
231
+ return client.search(terms, options);
232
+ }
233
+
234
+ private async searchCisaKev(terms: readonly string[], options: CisaKevSearchOptions): Promise<import('./types.ts').CisaKevResult> {
235
+ const client = this.config.cisaKevClient ?? createDefaultCisaKevClient();
236
+ return client.search(terms, options);
237
+ }
238
+
239
+ private async searchGitHub(terms: readonly string[], options: GitHubSearchOptions): Promise<import('./types.ts').GitHubResult> {
240
+ const client = this.config.githubAdvisoriesClient ?? createDefaultGitHubClient();
241
+ return client.search(terms, options);
242
+ }
243
+
244
+ private async searchOSV(terms: readonly string[], options: OSVSearchOptions): Promise<import('./types.ts').OSVResult> {
245
+ const client = this.config.osvClient ?? createDefaultOSVClient();
246
+ return client.search(terms, options);
247
+ }
248
+ }
249
+
250
+ // ============================================================================
251
+ // Default Client Implementations (wrappers for existing functions)
252
+ // ============================================================================
253
+
254
+ class DefaultNVDClient implements INVDClient {
255
+ async search(terms: readonly string[], options?: NVDSearchOptions): Promise<import('./types.ts').NVDResult> {
256
+ return searchNVD(terms as string[], options);
257
+ }
258
+
259
+ async getById(cveId: string): Promise<import('./types.ts').Vulnerability | null> {
260
+ const { getCVEById } = await import('./nvd.ts');
261
+ return getCVEById(cveId);
262
+ }
263
+ }
264
+
265
+ class DefaultCisaKevClient implements ICisaKevClient {
266
+ async search(terms: readonly string[], options?: CisaKevSearchOptions): Promise<import('./types.ts').CisaKevResult> {
267
+ return searchCisaKev(terms as string[], options);
268
+ }
269
+ }
270
+
271
+ class DefaultGitHubClient implements IGitHubAdvisoriesClient {
272
+ async search(terms: readonly string[], options?: GitHubSearchOptions): Promise<import('./types.ts').GitHubResult> {
273
+ return searchGitHubAdvisories(terms as string[], options);
274
+ }
275
+
276
+ async getById(id: string): Promise<import('./types.ts').Advisory | null> {
277
+ const { getAdvisoryById } = await import('./github-advisories.ts');
278
+ return getAdvisoryById(id);
279
+ }
280
+ }
281
+
282
+ class DefaultOSVClient implements IOSVClient {
283
+ async search(terms: readonly string[], options?: OSVSearchOptions): Promise<import('./types.ts').OSVResult> {
284
+ return searchOSV(terms as string[], options);
285
+ }
286
+
287
+ async getById(osvId: string): Promise<import('./types.ts').Vulnerability | null> {
288
+ const { getOSVById } = await import('./osv.ts');
289
+ return getOSVById(osvId);
290
+ }
291
+ }
292
+
293
+ // Default client factories
294
+ function createDefaultNVDClient(): INVDClient {
295
+ return new DefaultNVDClient();
296
+ }
297
+
298
+ function createDefaultCisaKevClient(): ICisaKevClient {
299
+ return new DefaultCisaKevClient();
300
+ }
301
+
302
+ function createDefaultGitHubClient(): IGitHubAdvisoriesClient {
303
+ return new DefaultGitHubClient();
304
+ }
305
+
306
+ function createDefaultOSVClient(): IOSVClient {
307
+ return new DefaultOSVClient();
308
+ }
309
+
310
+ // ============================================================================
311
+ // Global State for Backward Compatibility
312
+ // ============================================================================
313
+
314
+ let globalSearcher: SecuritySearcher | null = null;
315
+
316
+ /**
317
+ * Create a new security searcher instance
318
+ */
319
+ export function createSecuritySearcher(
320
+ config: Partial<SecuritySearcherConfig> = {}
321
+ ): SecuritySearcher {
322
+ return new SecuritySearcher(config);
323
+ }
324
+
325
+ /**
326
+ * Get the global security searcher instance
327
+ */
328
+ export function getSecuritySearcher(): SecuritySearcher {
329
+ if (!globalSearcher) {
330
+ globalSearcher = new SecuritySearcher();
331
+ }
332
+ return globalSearcher;
333
+ }
334
+
335
+ /**
336
+ * Set the global security searcher instance (for testing)
337
+ */
338
+ export function setSecuritySearcher(searcher: SecuritySearcher | null): void {
339
+ globalSearcher = searcher;
340
+ }
341
+
342
+ /**
343
+ * Reset the global security searcher instance (for testing)
344
+ */
345
+ export function resetSecuritySearcher(): void {
346
+ globalSearcher = null;
347
+ }
348
+
349
+ // ============================================================================
350
+ // Backward Compatibility Functions
351
+ // ============================================================================
352
+
353
+ /**
354
+ * Main search function that queries multiple security databases
355
+ * @deprecated Use SecuritySearcher class directly for better testability
356
+ */
357
+ export async function searchSecurityDatabases(
358
+ params: SecuritySearchParams,
359
+ ): Promise<SecuritySearchResult> {
360
+ const searcher = getSecuritySearcher();
361
+ return searcher.search(params);
362
+ }
363
+
364
+ /**
365
+ * Get database info for UI
366
+ */
367
+ export function getDatabaseInfo(): DatabaseInfo {
368
+ return {
369
+ databases: [
370
+ {
371
+ id: 'nvd',
372
+ name: 'NIST NVD',
373
+ description: 'National Vulnerability Database - 340,000+ CVE records',
374
+ url: 'https://nvd.nist.gov',
375
+ free: true,
376
+ },
377
+ {
378
+ id: 'cisa_kev',
379
+ name: 'CISA KEV Catalog',
380
+ description: 'Known Exploited Vulnerabilities - actively exploited vulnerabilities',
381
+ url: 'https://www.cisa.gov/known-exploited-vulnerabilities-catalog',
382
+ free: true,
383
+ },
384
+ {
385
+ id: 'github',
386
+ name: 'GitHub Advisory Database',
387
+ description: 'GitHub Security Advisories - open source security advisories',
388
+ url: 'https://github.com/advisories',
389
+ free: true,
390
+ },
391
+ {
392
+ id: 'osv',
393
+ name: 'Open Source Vulnerabilities',
394
+ description: 'OSV - open source package vulnerabilities',
395
+ url: 'https://osv.dev',
396
+ free: true,
397
+ },
398
+ ] as const,
399
+ };
400
+ }
401
+
402
+ // Export individual database search functions for direct use
403
+ export { searchNVD } from './nvd.ts';
404
+ export { searchCisaKev } from './cisa-kev.ts';
405
+ export { searchGitHubAdvisories } from './github-advisories.ts';
406
+ export { searchOSV } from './osv.ts';