nosible 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +438 -0
  2. package/dist/index.cjs +1841 -0
  3. package/dist/index.cjs.map +29 -0
  4. package/dist/index.js +1815 -0
  5. package/dist/index.js.map +29 -0
  6. package/package.json +63 -0
  7. package/src/api/api.test.ts +366 -0
  8. package/src/api/index.ts +179 -0
  9. package/src/api/schemas.ts +152 -0
  10. package/src/client.test.ts +685 -0
  11. package/src/client.ts +762 -0
  12. package/src/index.ts +4 -0
  13. package/src/scrape/types.ts +119 -0
  14. package/src/scrape/webPageData.test.ts +302 -0
  15. package/src/scrape/webPageData.ts +103 -0
  16. package/src/search/analyze.test.ts +396 -0
  17. package/src/search/analyze.ts +151 -0
  18. package/src/search/bulkSearch.ts +62 -0
  19. package/src/search/result.test.ts +423 -0
  20. package/src/search/result.ts +391 -0
  21. package/src/search/result.types.ts +32 -0
  22. package/src/search/resultFactory.ts +21 -0
  23. package/src/search/resultSet.io.test.ts +320 -0
  24. package/src/search/resultSet.test.ts +368 -0
  25. package/src/search/resultSet.ts +387 -0
  26. package/src/search/resultSet.types.ts +3 -0
  27. package/src/search/search.test.ts +299 -0
  28. package/src/search/search.ts +187 -0
  29. package/src/search/searchSet.io.test.ts +321 -0
  30. package/src/search/searchSet.ts +122 -0
  31. package/src/search/sqlFilter.test.ts +129 -0
  32. package/src/search/sqlFilter.ts +147 -0
  33. package/src/test-utils/mocks.ts +159 -0
  34. package/src/topicTrend/topicTrend.ts +53 -0
  35. package/src/utils/browser.test.ts +209 -0
  36. package/src/utils/browser.ts +21 -0
  37. package/src/utils/fernet.ts +47 -0
  38. package/src/utils/file.test.ts +81 -0
  39. package/src/utils/file.ts +195 -0
  40. package/src/utils/index.ts +7 -0
  41. package/src/utils/llm.test.ts +279 -0
  42. package/src/utils/llm.ts +244 -0
  43. package/src/utils/userPlan.test.ts +332 -0
  44. package/src/utils/userPlan.ts +211 -0
package/src/client.ts ADDED
@@ -0,0 +1,762 @@
1
+ /**
2
+ * Nosible AI Search API Client
3
+ *
4
+ * A comprehensive TypeScript client for interacting with the Nosible AI-powered search engine.
5
+ * This client provides fast, comprehensive, and bulk search capabilities across web-scale data,
6
+ * along with URL scraping and topic trend analysis features.
7
+ *
8
+ * @packageDocumentation
9
+ *
10
+ * @example
11
+ * ```typescript
12
+ * import { NosibleClient } from 'nosible-js';
13
+ *
14
+ * // Initialize with API key
15
+ * const client = new NosibleClient('your-api-key');
16
+ *
17
+ * // Or use environment variable NOSIBLE_API_KEY
18
+ * const client = new NosibleClient();
19
+ *
20
+ * // Perform a fast search
21
+ * const results = await client.fastSearch({
22
+ * question: 'Who has invested in Nosible?',
23
+ * nResults: 15
24
+ * });
25
+ *
26
+ * // Perform AI-powered search
27
+ * const aiResults = await client.aiSearch({
28
+ * question: 'What are the latest trends in AI technology?',
29
+ * instruction: 'Focus on recent developments and breakthrough innovations'
30
+ * });
31
+ *
32
+ * // Scrape a URL
33
+ * const webpage = await client.scrapeUrl('https://example.com');
34
+ *
35
+ * // Analyze topic trends
36
+ * const trends = await client.topicTrend('artificial intelligence');
37
+ * ```
38
+ */
39
+
40
+ import {type SqlFilter} from "./search/sqlFilter";
41
+ import {
42
+ Search,
43
+ userSearchParamsSchema,
44
+ clientDefaultSearchParamsSchema,
45
+ type ClientDefaultSearchParamsType,
46
+ type ClientSearchParamsSchema,
47
+ type UserAiSearchParamsType,
48
+ type UserSearchParamsType,
49
+ } from "./search/search";
50
+ import {WebPageData} from "./scrape/webPageData";
51
+ import {TopicTrend} from "./topicTrend/topicTrend";
52
+ import type {BulkResType, ScrapeFullRes, SearchResType} from "./api/schemas";
53
+ import {getLimiters, getUserPlan, type Plan} from "./utils/userPlan";
54
+ import type Bottleneck from "bottleneck";
55
+ import {OpenAI} from "openai";
56
+ import z from "zod";
57
+ import {createResultSet} from "./search/resultFactory";
58
+ import {createResult} from "./search/resultFactory";
59
+ import type {ResultSet} from "./search/resultSet";
60
+ import {bulkSearchDownload} from "./search/bulkSearch";
61
+ import {
62
+ callNosibleApi,
63
+ validateFastSearchParams,
64
+ validateBulkSearchParams,
65
+ } from "./api/index";
66
+ import {SearchSet} from "./search/searchSet";
67
+
68
+ const fastSearchesParamsSchema = z.union([
69
+ userSearchParamsSchema.omit({question: true}).extend({
70
+ questions: z.array(z.string()),
71
+ }),
72
+ z.instanceof(SearchSet),
73
+ ]);
74
+
75
+ export type FastSearchesParamsType = z.infer<typeof fastSearchesParamsSchema>;
76
+
77
+ /**
78
+ * Nosible API Client
79
+ *
80
+ * Main client class for interacting with the Nosible AI Search API.
81
+ * Provides methods for fast search, standard search, and bulk search operations.
82
+ *
83
+ * @example
84
+ * ```typescript
85
+ * // Initialize with API key
86
+ * const client = new Nosible('your-api-key');
87
+ *
88
+ * // Or use environment variable NOSIBLE_API_KEY
89
+ * const client = new Nosible();
90
+ *
91
+ * // Perform a fast search
92
+ * const results = await client.fastSearch({
93
+ * question: 'Who has invested in Nosible?',
94
+ * nResults: 15
95
+ * });
96
+ * ```
97
+ */
98
+ const nosibleClientParams = z.union([
99
+ z.string(),
100
+ z.object({
101
+ apiKey: z.string(),
102
+ llmApiKey: z.string().optional(),
103
+ openAiBaseURL: z.string().optional(),
104
+ sentimentModel: z.string().optional(),
105
+ expansionsModel: z.string().optional(),
106
+ searchDefaults: clientDefaultSearchParamsSchema.optional(),
107
+ }),
108
+ ]);
109
+
110
+ /**
111
+ * Configuration options for initializing the NosibleClient.
112
+ *
113
+ * Can be provided as a string (API key only) or as an object with detailed configuration.
114
+ *
115
+ * @example
116
+ * ```typescript
117
+ * // Simple string API key
118
+ * const client = new NosibleClient('your-api-key');
119
+ *
120
+ * // Object with both API keys
121
+ * const client = new NosibleClient({
122
+ * apiKey: 'your-nosible-api-key',
123
+ * llmApiKey: 'your-openrouter-api-key' // Optional, for LLM features
124
+ * });
125
+ * ```
126
+ */
127
+ export type NosibleClientParams = z.infer<typeof nosibleClientParams>;
128
+ /**
129
+ * Nosible API Client
130
+ *
131
+ * The main client class for interacting with the Nosible AI Search API. This class provides
132
+ * a comprehensive interface for performing various types of searches, scraping web content,
133
+ * and analyzing topic trends.
134
+ *
135
+ * Features:
136
+ * - **Fast Search**: Quick, optimized searches with configurable result limits (10-100 results)
137
+ * - **AI Search**: Advanced AI-powered searches with custom instructions
138
+ * - **Bulk Search**: Large-scale searches for extensive data collection (1000-10000 results)
139
+ * - **URL Scraping**: Extract structured content from web pages
140
+ * - **Topic Trends**: Analyze trending topics and their popularity over time
141
+ *
142
+ * The client automatically handles rate limiting, request validation, and response parsing.
143
+ * It also supports optional OpenRouter integration for enhanced LLM capabilities.
144
+ *
145
+ * @example
146
+ * ```typescript
147
+ * // Initialize with API key
148
+ * const client = new NosibleClient('your-api-key');
149
+ *
150
+ * // Or use environment variable NOSIBLE_API_KEY
151
+ * const client = new NosibleClient();
152
+ *
153
+ * // Initialize with default search parameters
154
+ * const clientWithDefaults = new NosibleClient({
155
+ * apiKey: 'your-api-key',
156
+ * searchDefaults: {
157
+ * nResults: 20,
158
+ * algorithm: 'hybrid-2',
159
+ * continent: 'North America'
160
+ * }
161
+ * });
162
+ *
163
+ * // Perform a fast search (will use defaults from client)
164
+ * const results = await clientWithDefaults.fastSearch({
165
+ * question: 'Who has invested in Nosible?'
166
+ * // nResults, algorithm, and continent will use defaults
167
+ * });
168
+ *
169
+ * // Perform a fast search
170
+ * const results = await client.fastSearch({
171
+ * question: 'Who has invested in Nosible?',
172
+ * nResults: 15
173
+ * });
174
+ *
175
+ * // Access search results
176
+ * console.log(`Found ${results.length} results`);
177
+ * results.forEach(result => {
178
+ * console.log(`Title: ${result.title}`);
179
+ * console.log(`URL: ${result.url}`);
180
+ * console.log(`Content: ${result.content.substring(0, 200)}...`);
181
+ * });
182
+ * ```
183
+ */
184
+ export class NosibleClient {
185
+ /** Base URL for the Nosible API */
186
+ private baseUrl: string = "https://www.nosible.ai/search/v2";
187
+ /** Your Nosible API key for authentication */
188
+ private apiKey: string;
189
+ /** Optional OpenAI client for LLM integration via OpenRouter */
190
+ public llmClient: OpenAI | undefined;
191
+ /** LLM model to use for expansions */
192
+ public expansionsModel: string = "openai/gpt-4o";
193
+ /** Default search parameters to apply to all searches */
194
+ private searchDefaults: ClientDefaultSearchParamsType | undefined;
195
+ /** User plan information determining rate limits and quotas */
196
+ private plan: Plan;
197
+ /** Rate limiters for different API endpoints to prevent quota exceeded errors */
198
+ private limiters: {
199
+ /** Rate limiter for URL scraping operations */
200
+ scrapeUrl: Bottleneck;
201
+ /** Rate limiter for bulk search operations */
202
+ bulk: Bottleneck;
203
+ /** Rate limiter for fast search operations */
204
+ fast: Bottleneck;
205
+ };
206
+
207
+ /**
208
+ * Creates a new NosibleClient instance.
209
+ *
210
+ * The client can be initialized in several ways:
211
+ * - With a string API key
212
+ * - With an object containing API keys
213
+ * - With no parameters (uses environment variables)
214
+ *
215
+ * Environment variables:
216
+ * - `NOSIBLE_API_KEY`: Your Nosible API key (required if no params provided)
217
+ * - `LLM_API_KEY`: Optional OpenRouter API key for LLM features
218
+ *
219
+ * @param params - Configuration options. Can be a string API key or an object with detailed configuration.
220
+ *
221
+ * @throws {Error} When no valid API key is provided
222
+ *
223
+ * @example
224
+ * ```typescript
225
+ * // Initialize with string API key
226
+ * const client = new NosibleClient('your-api-key');
227
+ *
228
+ * // Initialize with object configuration
229
+ * const client = new NosibleClient({
230
+ * apiKey: 'your-nosible-api-key',
231
+ * llmApiKey: 'your-openrouter-api-key'
232
+ * });
233
+ *
234
+ * // Initialize with default search parameters
235
+ * const client = new NosibleClient({
236
+ * apiKey: 'your-nosible-api-key',
237
+ * searchDefaults: {
238
+ * nResults: 20,
239
+ * algorithm: 'hybrid-2',
240
+ * continent: 'North America'
241
+ * }
242
+ * });
243
+ *
244
+ * // Initialize using environment variables
245
+ * const client = new NosibleClient();
246
+ * ```
247
+ */
248
+ constructor(params?: NosibleClientParams) {
249
+ /** Extract API key from string parameter */
250
+ let apiKeyToUse: string | undefined;
251
+ /** Extract OpenRouter API key for optional LLM integration */
252
+ let llmApiKeyToUse: string | undefined;
253
+ let openAiBaseURL: string | undefined;
254
+ let searchDefaultsToUse: ClientDefaultSearchParamsType | undefined;
255
+
256
+ if (typeof params === "string") {
257
+ // string param provided
258
+ apiKeyToUse = params;
259
+ } else if (params) {
260
+ // object params provided
261
+ apiKeyToUse = params.apiKey;
262
+ llmApiKeyToUse = params.llmApiKey;
263
+ openAiBaseURL = params.openAiBaseURL;
264
+ searchDefaultsToUse = params.searchDefaults;
265
+ } else {
266
+ // no params provided - use environment variables
267
+ apiKeyToUse = process.env.NOSIBLE_API_KEY;
268
+ llmApiKeyToUse = process.env.LLM_API_KEY;
269
+ }
270
+
271
+ /** Validate and set the Nosible API key */
272
+ if (!apiKeyToUse) {
273
+ throw new Error("API key is required");
274
+ } else {
275
+ this.verifyAndSetApiKey(apiKeyToUse);
276
+ }
277
+
278
+ /** Initialize OpenAI client for LLM integration if OpenRouter API key is provided */
279
+ if (llmApiKeyToUse) {
280
+ this.llmClient = new OpenAI({
281
+ apiKey: llmApiKeyToUse,
282
+ baseURL: openAiBaseURL,
283
+ });
284
+ }
285
+
286
+ /** Store API key, default search params, and initialize rate limiters based on user plan */
287
+ this.apiKey = apiKeyToUse;
288
+ this.searchDefaults = searchDefaultsToUse;
289
+ this.plan = getUserPlan(this.apiKey);
290
+ this.limiters = getLimiters(this.apiKey);
291
+ }
292
+
293
+ /**
294
+ * Merges default search parameters with provided search parameters.
295
+ *
296
+ * Provided parameters take precedence over default parameters.
297
+ *
298
+ * @param params - The search parameters provided by the user
299
+ * @returns Merged search parameters with defaults applied
300
+ * @private
301
+ */
302
+ private mergeWithDefaultSearch(
303
+ params: UserSearchParamsType
304
+ ): UserSearchParamsType {
305
+ if (!this.searchDefaults) {
306
+ return params;
307
+ }
308
+
309
+ // Merge default search params with provided params
310
+ // Provided params take precedence over defaults
311
+ return {
312
+ ...this.searchDefaults,
313
+ ...params,
314
+ };
315
+ }
316
+
317
+ /**
318
+ * Validates and sets the API key for the client.
319
+ *
320
+ * The API key must be in the format "key_id|secret_key" with exactly one pipe separator.
321
+ * This format allows for key identification and authentication.
322
+ *
323
+ * @param apiKey - The API key to validate and set
324
+ * @throws {Error} When the API key is missing or has invalid format
325
+ * @private
326
+ */
327
+ private verifyAndSetApiKey(apiKey: string | undefined) {
328
+ if (!apiKey) {
329
+ throw new Error("API key is required");
330
+ }
331
+ /** Split API key to validate format (should be "key_id|secret_key") */
332
+ const splits = apiKey.split("|");
333
+ if (splits.length !== 2) {
334
+ throw new Error(
335
+ "API key is invalid - expected format 'key_id|secret_key'"
336
+ );
337
+ }
338
+ this.apiKey = apiKey;
339
+ }
340
+
341
+ /**
342
+ * Makes an authenticated HTTP request to the Nosible API.
343
+ *
344
+ * This private method handles all HTTP communication with the API, including
345
+ * authentication headers, request body serialization, and error handling.
346
+ *
347
+ * @param endpoint - The API endpoint to call (relative to baseUrl)
348
+ * @param body - The request body to send as JSON
349
+ * @returns The parsed JSON response from the API
350
+ * @throws {Error} When the HTTP request fails or returns an error status
351
+ * @private
352
+ */
353
+ private async _request(endpoint: string, body: any): Promise<any> {
354
+ return callNosibleApi({
355
+ endpoint,
356
+ body,
357
+ apiKey: this.apiKey,
358
+ });
359
+ }
360
+
361
+ /**
362
+ * Performs a fast search query with optimized performance.
363
+ *
364
+ * Fast search is designed for quick, efficient queries with smaller result sets.
365
+ * It's ideal for real-time applications, autocomplete features, or when you need
366
+ * rapid responses with a limited number of highly relevant results.
367
+ *
368
+ * Features:
369
+ * - Optimized for speed (10-100 results)
370
+ * - Supports advanced filtering and search algorithms
371
+ * - Rate limited to prevent quota exceeded errors
372
+ * - Returns structured ResultSet with metadata
373
+ *
374
+ * @param params - Search parameters including question, filters, and result count
375
+ * @returns Promise resolving to a ResultSet containing search results and metadata
376
+ *
377
+ * @example
378
+ * ```typescript
379
+ * // Basic fast search
380
+ * const results = await client.fastSearch({
381
+ * question: 'latest AI technology trends',
382
+ * nResults: 20
383
+ * });
384
+ *
385
+ * // Advanced fast search with filters
386
+ * const filteredResults = await client.fastSearch({
387
+ * question: 'machine learning startups',
388
+ * nResults: 50,
389
+ * algorithm: 'hybrid-2',
390
+ * minSimilarity: 0.7,
391
+ * mustInclude: ['artificial intelligence', 'funding'],
392
+ * mustExclude: ['cryptocurrency'],
393
+ * continent: 'North America'
394
+ * });
395
+ *
396
+ * // Process results
397
+ * console.log(`Found ${results.length} results`);
398
+ * for (const result of results) {
399
+ * console.log(`Title: ${result.title}`);
400
+ * console.log(`URL: ${result.url}`);
401
+ * console.log(`Similarity: ${result.similarity}`);
402
+ * }
403
+ * ```
404
+ */
405
+ public async fastSearch(
406
+ params: ClientSearchParamsSchema
407
+ ): Promise<ResultSet> {
408
+ let search: Search;
409
+
410
+ // Type guard to check if we have a Search instance
411
+ const hasSearchInstance = (
412
+ params: ClientSearchParamsSchema
413
+ ): params is {search: Search} => {
414
+ return "search" in params;
415
+ };
416
+
417
+ if (hasSearchInstance(params)) {
418
+ // Use provided Search instance
419
+ if (params.search instanceof Search) {
420
+ search = params.search;
421
+ } else {
422
+ throw new Error("Invalid search parameter provided.");
423
+ }
424
+ } else {
425
+ // Create new Search instance from params
426
+ // Merge with default search parameters before creating Search instance
427
+ const mergedParams = this.mergeWithDefaultSearch(params);
428
+ search = new Search(mergedParams);
429
+ }
430
+
431
+ /** Generate search body from Search instance */
432
+ const body = await search.searchBody({client: this});
433
+
434
+ /** Validate search body */
435
+ const validatedBody = validateFastSearchParams(body);
436
+
437
+ /** Make API request and parse response */
438
+ const {message, query, response} = (await this._request(
439
+ "fast-search",
440
+ validatedBody
441
+ )) as SearchResType;
442
+
443
+ const results = response.map((result) => {
444
+ const fullResult = {...query, ...result};
445
+ return fullResult;
446
+ });
447
+ /** Wrap response in ResultSet for enhanced functionality */
448
+ const resultSet = createResultSet(this, results);
449
+ return resultSet;
450
+ }
451
+
452
+ public async fastSearches(
453
+ params: FastSearchesParamsType
454
+ ): Promise<ResultSet[]> {
455
+ let searchSet: SearchSet;
456
+
457
+ // Handle SearchSet input
458
+ if (params instanceof SearchSet) {
459
+ searchSet = params;
460
+ } else {
461
+ // Handle questions array with shared search parameters
462
+ const {questions, ...sharedParams} = params;
463
+
464
+ // Merge shared parameters with default search parameters
465
+ const mergedSharedParams = this.mergeWithDefaultSearch(sharedParams);
466
+
467
+ // Create Search instances for each question using merged shared parameters
468
+ const searches = questions.map(
469
+ (question) => new Search({question, ...mergedSharedParams})
470
+ );
471
+
472
+ searchSet = new SearchSet(searches);
473
+ }
474
+
475
+ // Execute fastSearch for all searches in parallel
476
+ const results = await Promise.all(
477
+ searchSet.searches.map((search) => this.fastSearch({search}))
478
+ );
479
+
480
+ return results;
481
+ }
482
+
483
+ /**
484
+ * Performs an AI-powered search with custom instructions.
485
+ *
486
+ * AI search leverages advanced language models to understand complex queries
487
+ * and custom instructions, providing more contextually relevant results.
488
+ * This method is ideal for nuanced searches that require understanding intent,
489
+ * context, or specific analytical requirements.
490
+ *
491
+ * Features:
492
+ * - Natural language understanding of complex queries
493
+ * - Custom instructions for specialized search behavior
494
+ * - Context-aware result ranking and filtering
495
+ * - Rate limited for optimal performance
496
+ *
497
+ * @param params - AI search parameters including question and custom instructions
498
+ * @returns Promise resolving to a ResultSet containing AI-enhanced search results
499
+ *
500
+ * @example
501
+ * ```typescript
502
+ * // Basic AI search
503
+ * const results = await client.aiSearch({
504
+ * question: 'What are the environmental impacts of renewable energy?',
505
+ * instruction: 'Focus on recent studies and quantitative data'
506
+ * });
507
+ *
508
+ * // AI search with specific analytical focus
509
+ * const analysisResults = await client.aiSearch({
510
+ * question: 'company financial performance',
511
+ * instruction: 'Compare revenue growth and profit margins across quarters',
512
+ * nResults: 25,
513
+ * algorithm: 'hybrid-3'
514
+ * });
515
+ *
516
+ * // AI search for competitive analysis
517
+ * const competitiveResults = await client.aiSearch({
518
+ * question: 'competitors in the cloud computing market',
519
+ * instruction: 'Identify key players, market share, and competitive advantages'
520
+ * });
521
+ * ```
522
+ */
523
+ public async aiSearch(params: UserAiSearchParamsType): Promise<ResultSet> {
524
+ /** Execute AI search within rate limiter */
525
+ /** Make direct API request with AI search parameters */
526
+ const json = (await this._request("search", params)) as SearchResType;
527
+ /** Wrap response in ResultSet for enhanced functionality */
528
+
529
+ const results = json.response.map((result) => {
530
+ const fullResult = {...json.query, ...result};
531
+ return fullResult;
532
+ });
533
+ const resultSet = createResultSet(this, results);
534
+ return resultSet;
535
+ }
536
+
537
+ /**
538
+ * Performs a bulk search for large-scale data collection.
539
+ *
540
+ * Bulk search is designed for comprehensive data gathering and analysis tasks.
541
+ * It supports large result sets (1000-10000 results) and provides efficient
542
+ * download mechanisms for extensive datasets. Results are encrypted and
543
+ * downloaded asynchronously for optimal performance.
544
+ *
545
+ * Features:
546
+ * - Large-scale data collection (1000-10000 results)
547
+ * - Encrypted result delivery for security
548
+ * - Asynchronous download processing
549
+ * - Comprehensive filtering and search options
550
+ * - Rate limited for optimal server performance
551
+ * - Supports both raw parameters and Search instances
552
+ *
553
+ * @param params - Bulk search parameters including question or search instance, with comprehensive filtering options
554
+ * @returns Promise resolving to a ResultSet containing bulk search results
555
+ *
556
+ * @example
557
+ * ```typescript
558
+ * // Basic bulk search
559
+ * const results = await client.bulkSearch({
560
+ * question: 'all articles about artificial intelligence',
561
+ * nResults: 5000
562
+ * });
563
+ *
564
+ * // Bulk search with comprehensive filtering
565
+ * const filteredResults = await client.bulkSearch({
566
+ * question: 'technology company funding rounds',
567
+ * nResults: 10000,
568
+ * algorithm: 'hybrid-3',
569
+ * minSimilarity: 0.6,
570
+ * continent: 'North America',
571
+ * region: 'Silicon Valley',
572
+ * mustInclude: ['venture capital', 'funding', 'investment'],
573
+ * mustExclude: ['cryptocurrency', 'NFT'],
574
+ * nProbes: 8,
575
+ * nContextify: 512
576
+ * });
577
+ *
578
+ * // Bulk search using a Search instance
579
+ * const search = new Search({
580
+ * question: 'renewable energy developments',
581
+ * nResults: 7500,
582
+ * algorithm: 'hybrid-2'
583
+ * });
584
+ * const searchResults = await client.bulkSearch({ search });
585
+ *
586
+ * // Process bulk results
587
+ * console.log(`Retrieved ${results.length} results`);
588
+ * const data = results.toJSON(); // Convert to JSON for analysis
589
+ * ```
590
+ */
591
+ public async bulkSearch(
592
+ params: ClientSearchParamsSchema
593
+ ): Promise<ResultSet> {
594
+ let search: Search;
595
+
596
+ // Type guard to check if we have a Search instance
597
+ const hasSearchInstance = (
598
+ params: ClientSearchParamsSchema
599
+ ): params is {search: Search} => {
600
+ return "search" in params;
601
+ };
602
+
603
+ if (hasSearchInstance(params)) {
604
+ // Use provided Search instance
605
+ if (params.search instanceof Search) {
606
+ search = params.search;
607
+ } else {
608
+ throw new Error("Invalid search parameter provided.");
609
+ }
610
+ } else {
611
+ // Create new Search instance from params
612
+ // Merge with default search parameters before creating Search instance
613
+ const mergedParams = this.mergeWithDefaultSearch(params);
614
+ search = new Search(mergedParams);
615
+ }
616
+
617
+ /** Generate search body from Search instance */
618
+ const body = await search.searchBody({client: this});
619
+
620
+ /** Validate search body for bulk search */
621
+ const validatedBody = validateBulkSearchParams(body);
622
+
623
+ /** Request bulk search and get encrypted file location */
624
+ const fileLocation = (await this._request(
625
+ "bulk-search",
626
+ validatedBody
627
+ )) as BulkResType;
628
+
629
+ /** Download and decrypt the bulk search results */
630
+ const json = (await bulkSearchDownload({
631
+ downloadFrom: fileLocation.download_from,
632
+ decryptUsing: fileLocation.decrypt_using,
633
+ })) as SearchResType;
634
+ const results = json.response.map((result) => {
635
+ const fullResult = {...json.query, ...result};
636
+ return fullResult;
637
+ });
638
+ /** Wrap response in ResultSet for enhanced functionality */
639
+ const resultSet = createResultSet(this, results);
640
+ return resultSet;
641
+ }
642
+
643
+ /**
644
+ * Scrapes and extracts structured content from a web URL.
645
+ *
646
+ * This method fetches the content of a web page and extracts structured information
647
+ * including title, description, main content, metadata, and other relevant details.
648
+ * The scraping is optimized for clean content extraction and handles various
649
+ * website structures and formats.
650
+ *
651
+ * Features:
652
+ * - Clean content extraction from web pages
653
+ * - Structured data including metadata and content
654
+ * - Handles various website formats and structures
655
+ * - Rate limited to respect website policies
656
+ * - Returns WebPage object with enhanced functionality
657
+ *
658
+ * @param url - The URL to scrape and extract content from
659
+ * @returns Promise resolving to a WebPage object containing structured content
660
+ *
661
+ * @example
662
+ * ```typescript
663
+ * // Basic URL scraping
664
+ * const webpage = await client.scrapeUrl('https://example.com/article');
665
+ *
666
+ * // Access scraped content
667
+ * console.log(`Title: ${webpage.title}`);
668
+ * console.log(`Description: ${webpage.description}`);
669
+ * console.log(`Author: ${webpage.author}`);
670
+ * console.log(`Published: ${webpage.published}`);
671
+ * console.log(`Content length: ${webpage.content.length} characters`);
672
+ *
673
+ * // Extract specific content sections
674
+ * const mainContent = webpage.content;
675
+ * const bestChunk = webpage.bestChunk; // Most relevant content chunk
676
+ *
677
+ * // Use in combination with search
678
+ * const searchResults = await client.fastSearch({
679
+ * question: 'artificial intelligence news',
680
+ * nResults: 10
681
+ * });
682
+ *
683
+ * // Scrape the top result
684
+ * if (searchResults.length > 0) {
685
+ * const topArticle = await client.scrapeUrl(searchResults[0].url);
686
+ * console.log(`Full article: ${topArticle.content}`);
687
+ * }
688
+ * ```
689
+ */
690
+ public async scrapeUrl(url: string): Promise<WebPageData> {
691
+ /** Execute URL scraping within rate limiter */
692
+ return this.limiters.scrapeUrl.schedule(async () => {
693
+ /** Make API request to scrape the URL */
694
+ const json = (await this._request("scrape-url", {url})) as ScrapeFullRes;
695
+ /** Wrap response in WebPage object for enhanced functionality */
696
+ const webPage = new WebPageData(this, json.response);
697
+ return webPage;
698
+ });
699
+ }
700
+
701
+ /**
702
+ * Analyzes topic trends and popularity over time.
703
+ *
704
+ * This method provides insights into how a topic's popularity and relevance
705
+ * changes over time. It's useful for market research, content strategy,
706
+ * trend analysis, and understanding topic momentum.
707
+ *
708
+ * Features:
709
+ * - Historical trend analysis for any topic
710
+ * - Popularity metrics and temporal patterns
711
+ * - Optional SQL filtering for refined analysis
712
+ * - Returns TopicTrend object with analytical data
713
+ *
714
+ * @param query - The topic or query to analyze trends for
715
+ * @param sqlFilter - Optional SQL filter to refine the trend analysis
716
+ * @returns Promise resolving to a TopicTrend object containing trend analysis data
717
+ *
718
+ * @example
719
+ * ```typescript
720
+ * // Basic trend analysis
721
+ * const trends = await client.topicTrend('artificial intelligence');
722
+ *
723
+ * // Access trend data
724
+ * console.log(`Topic: ${trends.query}`);
725
+ * console.log(`Data points: ${trends.data.length}`);
726
+ * trends.data.forEach(point => {
727
+ * console.log(`Date: ${point.date}, Popularity: ${point.popularity}`);
728
+ * });
729
+ *
730
+ * // Trend analysis with filtering
731
+ * const filteredTrends = await client.topicTrend(
732
+ * 'machine learning',
733
+ * 'continent = "North America" AND published >= "2023-01-01"'
734
+ * );
735
+ *
736
+ * // Analyze multiple related topics
737
+ * const topics = ['AI', 'machine learning', 'deep learning', 'neural networks'];
738
+ * for (const topic of topics) {
739
+ * const trend = await client.topicTrend(topic);
740
+ * console.log(`${topic} trend analysis complete`);
741
+ * }
742
+ *
743
+ * // Use trend data for content strategy
744
+ * const trendingTopics = ['blockchain', 'quantum computing', 'renewable energy'];
745
+ * const trendAnalyses = await Promise.all(
746
+ * trendingTopics.map(topic => client.topicTrend(topic))
747
+ * );
748
+ * ```
749
+ */
750
+ public async topicTrend(
751
+ query: string,
752
+ sqlFilter?: SqlFilter
753
+ ): Promise<TopicTrend> {
754
+ /** Prepare request body with query and optional SQL filter */
755
+ const body = {query, sql_filter: sqlFilter};
756
+ /** Make API request for trend analysis */
757
+ const json = await this._request("topic-trend", body);
758
+ /** Wrap response in TopicTrend object for enhanced functionality */
759
+ const topicTrend = new TopicTrend(json);
760
+ return topicTrend;
761
+ }
762
+ }