nosible 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +438 -0
- package/dist/index.cjs +1841 -0
- package/dist/index.cjs.map +29 -0
- package/dist/index.js +1815 -0
- package/dist/index.js.map +29 -0
- package/package.json +63 -0
- package/src/api/api.test.ts +366 -0
- package/src/api/index.ts +179 -0
- package/src/api/schemas.ts +152 -0
- package/src/client.test.ts +685 -0
- package/src/client.ts +762 -0
- package/src/index.ts +4 -0
- package/src/scrape/types.ts +119 -0
- package/src/scrape/webPageData.test.ts +302 -0
- package/src/scrape/webPageData.ts +103 -0
- package/src/search/analyze.test.ts +396 -0
- package/src/search/analyze.ts +151 -0
- package/src/search/bulkSearch.ts +62 -0
- package/src/search/result.test.ts +423 -0
- package/src/search/result.ts +391 -0
- package/src/search/result.types.ts +32 -0
- package/src/search/resultFactory.ts +21 -0
- package/src/search/resultSet.io.test.ts +320 -0
- package/src/search/resultSet.test.ts +368 -0
- package/src/search/resultSet.ts +387 -0
- package/src/search/resultSet.types.ts +3 -0
- package/src/search/search.test.ts +299 -0
- package/src/search/search.ts +187 -0
- package/src/search/searchSet.io.test.ts +321 -0
- package/src/search/searchSet.ts +122 -0
- package/src/search/sqlFilter.test.ts +129 -0
- package/src/search/sqlFilter.ts +147 -0
- package/src/test-utils/mocks.ts +159 -0
- package/src/topicTrend/topicTrend.ts +53 -0
- package/src/utils/browser.test.ts +209 -0
- package/src/utils/browser.ts +21 -0
- package/src/utils/fernet.ts +47 -0
- package/src/utils/file.test.ts +81 -0
- package/src/utils/file.ts +195 -0
- package/src/utils/index.ts +7 -0
- package/src/utils/llm.test.ts +279 -0
- package/src/utils/llm.ts +244 -0
- package/src/utils/userPlan.test.ts +332 -0
- package/src/utils/userPlan.ts +211 -0
package/src/client.ts
ADDED
|
@@ -0,0 +1,762 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nosible AI Search API Client
|
|
3
|
+
*
|
|
4
|
+
* A comprehensive TypeScript client for interacting with the Nosible AI-powered search engine.
|
|
5
|
+
* This client provides fast, comprehensive, and bulk search capabilities across web-scale data,
|
|
6
|
+
* along with URL scraping and topic trend analysis features.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* import { NosibleClient } from 'nosible-js';
|
|
13
|
+
*
|
|
14
|
+
* // Initialize with API key
|
|
15
|
+
* const client = new NosibleClient('your-api-key');
|
|
16
|
+
*
|
|
17
|
+
* // Or use environment variable NOSIBLE_API_KEY
|
|
18
|
+
* const client = new NosibleClient();
|
|
19
|
+
*
|
|
20
|
+
* // Perform a fast search
|
|
21
|
+
* const results = await client.fastSearch({
|
|
22
|
+
* question: 'Who has invested in Nosible?',
|
|
23
|
+
* nResults: 15
|
|
24
|
+
* });
|
|
25
|
+
*
|
|
26
|
+
* // Perform AI-powered search
|
|
27
|
+
* const aiResults = await client.aiSearch({
|
|
28
|
+
* question: 'What are the latest trends in AI technology?',
|
|
29
|
+
* instruction: 'Focus on recent developments and breakthrough innovations'
|
|
30
|
+
* });
|
|
31
|
+
*
|
|
32
|
+
* // Scrape a URL
|
|
33
|
+
* const webpage = await client.scrapeUrl('https://example.com');
|
|
34
|
+
*
|
|
35
|
+
* // Analyze topic trends
|
|
36
|
+
* const trends = await client.topicTrend('artificial intelligence');
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import {type SqlFilter} from "./search/sqlFilter";
|
|
41
|
+
import {
|
|
42
|
+
Search,
|
|
43
|
+
userSearchParamsSchema,
|
|
44
|
+
clientDefaultSearchParamsSchema,
|
|
45
|
+
type ClientDefaultSearchParamsType,
|
|
46
|
+
type ClientSearchParamsSchema,
|
|
47
|
+
type UserAiSearchParamsType,
|
|
48
|
+
type UserSearchParamsType,
|
|
49
|
+
} from "./search/search";
|
|
50
|
+
import {WebPageData} from "./scrape/webPageData";
|
|
51
|
+
import {TopicTrend} from "./topicTrend/topicTrend";
|
|
52
|
+
import type {BulkResType, ScrapeFullRes, SearchResType} from "./api/schemas";
|
|
53
|
+
import {getLimiters, getUserPlan, type Plan} from "./utils/userPlan";
|
|
54
|
+
import type Bottleneck from "bottleneck";
|
|
55
|
+
import {OpenAI} from "openai";
|
|
56
|
+
import z from "zod";
|
|
57
|
+
import {createResultSet} from "./search/resultFactory";
|
|
58
|
+
import {createResult} from "./search/resultFactory";
|
|
59
|
+
import type {ResultSet} from "./search/resultSet";
|
|
60
|
+
import {bulkSearchDownload} from "./search/bulkSearch";
|
|
61
|
+
import {
|
|
62
|
+
callNosibleApi,
|
|
63
|
+
validateFastSearchParams,
|
|
64
|
+
validateBulkSearchParams,
|
|
65
|
+
} from "./api/index";
|
|
66
|
+
import {SearchSet} from "./search/searchSet";
|
|
67
|
+
|
|
68
|
+
const fastSearchesParamsSchema = z.union([
|
|
69
|
+
userSearchParamsSchema.omit({question: true}).extend({
|
|
70
|
+
questions: z.array(z.string()),
|
|
71
|
+
}),
|
|
72
|
+
z.instanceof(SearchSet),
|
|
73
|
+
]);
|
|
74
|
+
|
|
75
|
+
export type FastSearchesParamsType = z.infer<typeof fastSearchesParamsSchema>;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Nosible API Client
|
|
79
|
+
*
|
|
80
|
+
* Main client class for interacting with the Nosible AI Search API.
|
|
81
|
+
* Provides methods for fast search, standard search, and bulk search operations.
|
|
82
|
+
*
|
|
83
|
+
* @example
|
|
84
|
+
* ```typescript
|
|
85
|
+
* // Initialize with API key
|
|
86
|
+
* const client = new Nosible('your-api-key');
|
|
87
|
+
*
|
|
88
|
+
* // Or use environment variable NOSIBLE_API_KEY
|
|
89
|
+
* const client = new Nosible();
|
|
90
|
+
*
|
|
91
|
+
* // Perform a fast search
|
|
92
|
+
* const results = await client.fastSearch({
|
|
93
|
+
* question: 'Who has invested in Nosible?',
|
|
94
|
+
* nResults: 15
|
|
95
|
+
* });
|
|
96
|
+
* ```
|
|
97
|
+
*/
|
|
98
|
+
const nosibleClientParams = z.union([
|
|
99
|
+
z.string(),
|
|
100
|
+
z.object({
|
|
101
|
+
apiKey: z.string(),
|
|
102
|
+
llmApiKey: z.string().optional(),
|
|
103
|
+
openAiBaseURL: z.string().optional(),
|
|
104
|
+
sentimentModel: z.string().optional(),
|
|
105
|
+
expansionsModel: z.string().optional(),
|
|
106
|
+
searchDefaults: clientDefaultSearchParamsSchema.optional(),
|
|
107
|
+
}),
|
|
108
|
+
]);
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Configuration options for initializing the NosibleClient.
|
|
112
|
+
*
|
|
113
|
+
* Can be provided as a string (API key only) or as an object with detailed configuration.
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* ```typescript
|
|
117
|
+
* // Simple string API key
|
|
118
|
+
* const client = new NosibleClient('your-api-key');
|
|
119
|
+
*
|
|
120
|
+
* // Object with both API keys
|
|
121
|
+
* const client = new NosibleClient({
|
|
122
|
+
* apiKey: 'your-nosible-api-key',
|
|
123
|
+
* llmApiKey: 'your-openrouter-api-key' // Optional, for LLM features
|
|
124
|
+
* });
|
|
125
|
+
* ```
|
|
126
|
+
*/
|
|
127
|
+
export type NosibleClientParams = z.infer<typeof nosibleClientParams>;
|
|
128
|
+
/**
|
|
129
|
+
* Nosible API Client
|
|
130
|
+
*
|
|
131
|
+
* The main client class for interacting with the Nosible AI Search API. This class provides
|
|
132
|
+
* a comprehensive interface for performing various types of searches, scraping web content,
|
|
133
|
+
* and analyzing topic trends.
|
|
134
|
+
*
|
|
135
|
+
* Features:
|
|
136
|
+
* - **Fast Search**: Quick, optimized searches with configurable result limits (10-100 results)
|
|
137
|
+
* - **AI Search**: Advanced AI-powered searches with custom instructions
|
|
138
|
+
* - **Bulk Search**: Large-scale searches for extensive data collection (1000-10000 results)
|
|
139
|
+
* - **URL Scraping**: Extract structured content from web pages
|
|
140
|
+
* - **Topic Trends**: Analyze trending topics and their popularity over time
|
|
141
|
+
*
|
|
142
|
+
* The client automatically handles rate limiting, request validation, and response parsing.
|
|
143
|
+
* It also supports optional OpenRouter integration for enhanced LLM capabilities.
|
|
144
|
+
*
|
|
145
|
+
* @example
|
|
146
|
+
* ```typescript
|
|
147
|
+
* // Initialize with API key
|
|
148
|
+
* const client = new NosibleClient('your-api-key');
|
|
149
|
+
*
|
|
150
|
+
* // Or use environment variable NOSIBLE_API_KEY
|
|
151
|
+
* const client = new NosibleClient();
|
|
152
|
+
*
|
|
153
|
+
* // Initialize with default search parameters
|
|
154
|
+
* const clientWithDefaults = new NosibleClient({
|
|
155
|
+
* apiKey: 'your-api-key',
|
|
156
|
+
* searchDefaults: {
|
|
157
|
+
* nResults: 20,
|
|
158
|
+
* algorithm: 'hybrid-2',
|
|
159
|
+
* continent: 'North America'
|
|
160
|
+
* }
|
|
161
|
+
* });
|
|
162
|
+
*
|
|
163
|
+
* // Perform a fast search (will use defaults from client)
|
|
164
|
+
* const results = await clientWithDefaults.fastSearch({
|
|
165
|
+
* question: 'Who has invested in Nosible?'
|
|
166
|
+
* // nResults, algorithm, and continent will use defaults
|
|
167
|
+
* });
|
|
168
|
+
*
|
|
169
|
+
* // Perform a fast search
|
|
170
|
+
* const results = await client.fastSearch({
|
|
171
|
+
* question: 'Who has invested in Nosible?',
|
|
172
|
+
* nResults: 15
|
|
173
|
+
* });
|
|
174
|
+
*
|
|
175
|
+
* // Access search results
|
|
176
|
+
* console.log(`Found ${results.length} results`);
|
|
177
|
+
* results.forEach(result => {
|
|
178
|
+
* console.log(`Title: ${result.title}`);
|
|
179
|
+
* console.log(`URL: ${result.url}`);
|
|
180
|
+
* console.log(`Content: ${result.content.substring(0, 200)}...`);
|
|
181
|
+
* });
|
|
182
|
+
* ```
|
|
183
|
+
*/
|
|
184
|
+
export class NosibleClient {
|
|
185
|
+
/** Base URL for the Nosible API */
|
|
186
|
+
private baseUrl: string = "https://www.nosible.ai/search/v2";
|
|
187
|
+
/** Your Nosible API key for authentication */
|
|
188
|
+
private apiKey: string;
|
|
189
|
+
/** Optional OpenAI client for LLM integration via OpenRouter */
|
|
190
|
+
public llmClient: OpenAI | undefined;
|
|
191
|
+
/** LLM model to use for expansions */
|
|
192
|
+
public expansionsModel: string = "openai/gpt-4o";
|
|
193
|
+
/** Default search parameters to apply to all searches */
|
|
194
|
+
private searchDefaults: ClientDefaultSearchParamsType | undefined;
|
|
195
|
+
/** User plan information determining rate limits and quotas */
|
|
196
|
+
private plan: Plan;
|
|
197
|
+
/** Rate limiters for different API endpoints to prevent quota exceeded errors */
|
|
198
|
+
private limiters: {
|
|
199
|
+
/** Rate limiter for URL scraping operations */
|
|
200
|
+
scrapeUrl: Bottleneck;
|
|
201
|
+
/** Rate limiter for bulk search operations */
|
|
202
|
+
bulk: Bottleneck;
|
|
203
|
+
/** Rate limiter for fast search operations */
|
|
204
|
+
fast: Bottleneck;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Creates a new NosibleClient instance.
|
|
209
|
+
*
|
|
210
|
+
* The client can be initialized in several ways:
|
|
211
|
+
* - With a string API key
|
|
212
|
+
* - With an object containing API keys
|
|
213
|
+
* - With no parameters (uses environment variables)
|
|
214
|
+
*
|
|
215
|
+
* Environment variables:
|
|
216
|
+
* - `NOSIBLE_API_KEY`: Your Nosible API key (required if no params provided)
|
|
217
|
+
* - `LLM_API_KEY`: Optional OpenRouter API key for LLM features
|
|
218
|
+
*
|
|
219
|
+
* @param params - Configuration options. Can be a string API key or an object with detailed configuration.
|
|
220
|
+
*
|
|
221
|
+
* @throws {Error} When no valid API key is provided
|
|
222
|
+
*
|
|
223
|
+
* @example
|
|
224
|
+
* ```typescript
|
|
225
|
+
* // Initialize with string API key
|
|
226
|
+
* const client = new NosibleClient('your-api-key');
|
|
227
|
+
*
|
|
228
|
+
* // Initialize with object configuration
|
|
229
|
+
* const client = new NosibleClient({
|
|
230
|
+
* apiKey: 'your-nosible-api-key',
|
|
231
|
+
* llmApiKey: 'your-openrouter-api-key'
|
|
232
|
+
* });
|
|
233
|
+
*
|
|
234
|
+
* // Initialize with default search parameters
|
|
235
|
+
* const client = new NosibleClient({
|
|
236
|
+
* apiKey: 'your-nosible-api-key',
|
|
237
|
+
* searchDefaults: {
|
|
238
|
+
* nResults: 20,
|
|
239
|
+
* algorithm: 'hybrid-2',
|
|
240
|
+
* continent: 'North America'
|
|
241
|
+
* }
|
|
242
|
+
* });
|
|
243
|
+
*
|
|
244
|
+
* // Initialize using environment variables
|
|
245
|
+
* const client = new NosibleClient();
|
|
246
|
+
* ```
|
|
247
|
+
*/
|
|
248
|
+
constructor(params?: NosibleClientParams) {
|
|
249
|
+
/** Extract API key from string parameter */
|
|
250
|
+
let apiKeyToUse: string | undefined;
|
|
251
|
+
/** Extract OpenRouter API key for optional LLM integration */
|
|
252
|
+
let llmApiKeyToUse: string | undefined;
|
|
253
|
+
let openAiBaseURL: string | undefined;
|
|
254
|
+
let searchDefaultsToUse: ClientDefaultSearchParamsType | undefined;
|
|
255
|
+
|
|
256
|
+
if (typeof params === "string") {
|
|
257
|
+
// string param provided
|
|
258
|
+
apiKeyToUse = params;
|
|
259
|
+
} else if (params) {
|
|
260
|
+
// object params provided
|
|
261
|
+
apiKeyToUse = params.apiKey;
|
|
262
|
+
llmApiKeyToUse = params.llmApiKey;
|
|
263
|
+
openAiBaseURL = params.openAiBaseURL;
|
|
264
|
+
searchDefaultsToUse = params.searchDefaults;
|
|
265
|
+
} else {
|
|
266
|
+
// no params provided - use environment variables
|
|
267
|
+
apiKeyToUse = process.env.NOSIBLE_API_KEY;
|
|
268
|
+
llmApiKeyToUse = process.env.LLM_API_KEY;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Validate and set the Nosible API key */
|
|
272
|
+
if (!apiKeyToUse) {
|
|
273
|
+
throw new Error("API key is required");
|
|
274
|
+
} else {
|
|
275
|
+
this.verifyAndSetApiKey(apiKeyToUse);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** Initialize OpenAI client for LLM integration if OpenRouter API key is provided */
|
|
279
|
+
if (llmApiKeyToUse) {
|
|
280
|
+
this.llmClient = new OpenAI({
|
|
281
|
+
apiKey: llmApiKeyToUse,
|
|
282
|
+
baseURL: openAiBaseURL,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Store API key, default search params, and initialize rate limiters based on user plan */
|
|
287
|
+
this.apiKey = apiKeyToUse;
|
|
288
|
+
this.searchDefaults = searchDefaultsToUse;
|
|
289
|
+
this.plan = getUserPlan(this.apiKey);
|
|
290
|
+
this.limiters = getLimiters(this.apiKey);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Merges default search parameters with provided search parameters.
|
|
295
|
+
*
|
|
296
|
+
* Provided parameters take precedence over default parameters.
|
|
297
|
+
*
|
|
298
|
+
* @param params - The search parameters provided by the user
|
|
299
|
+
* @returns Merged search parameters with defaults applied
|
|
300
|
+
* @private
|
|
301
|
+
*/
|
|
302
|
+
private mergeWithDefaultSearch(
|
|
303
|
+
params: UserSearchParamsType
|
|
304
|
+
): UserSearchParamsType {
|
|
305
|
+
if (!this.searchDefaults) {
|
|
306
|
+
return params;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Merge default search params with provided params
|
|
310
|
+
// Provided params take precedence over defaults
|
|
311
|
+
return {
|
|
312
|
+
...this.searchDefaults,
|
|
313
|
+
...params,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Validates and sets the API key for the client.
|
|
319
|
+
*
|
|
320
|
+
* The API key must be in the format "key_id|secret_key" with exactly one pipe separator.
|
|
321
|
+
* This format allows for key identification and authentication.
|
|
322
|
+
*
|
|
323
|
+
* @param apiKey - The API key to validate and set
|
|
324
|
+
* @throws {Error} When the API key is missing or has invalid format
|
|
325
|
+
* @private
|
|
326
|
+
*/
|
|
327
|
+
private verifyAndSetApiKey(apiKey: string | undefined) {
|
|
328
|
+
if (!apiKey) {
|
|
329
|
+
throw new Error("API key is required");
|
|
330
|
+
}
|
|
331
|
+
/** Split API key to validate format (should be "key_id|secret_key") */
|
|
332
|
+
const splits = apiKey.split("|");
|
|
333
|
+
if (splits.length !== 2) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
"API key is invalid - expected format 'key_id|secret_key'"
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
this.apiKey = apiKey;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Makes an authenticated HTTP request to the Nosible API.
|
|
343
|
+
*
|
|
344
|
+
* This private method handles all HTTP communication with the API, including
|
|
345
|
+
* authentication headers, request body serialization, and error handling.
|
|
346
|
+
*
|
|
347
|
+
* @param endpoint - The API endpoint to call (relative to baseUrl)
|
|
348
|
+
* @param body - The request body to send as JSON
|
|
349
|
+
* @returns The parsed JSON response from the API
|
|
350
|
+
* @throws {Error} When the HTTP request fails or returns an error status
|
|
351
|
+
* @private
|
|
352
|
+
*/
|
|
353
|
+
private async _request(endpoint: string, body: any): Promise<any> {
|
|
354
|
+
return callNosibleApi({
|
|
355
|
+
endpoint,
|
|
356
|
+
body,
|
|
357
|
+
apiKey: this.apiKey,
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Performs a fast search query with optimized performance.
|
|
363
|
+
*
|
|
364
|
+
* Fast search is designed for quick, efficient queries with smaller result sets.
|
|
365
|
+
* It's ideal for real-time applications, autocomplete features, or when you need
|
|
366
|
+
* rapid responses with a limited number of highly relevant results.
|
|
367
|
+
*
|
|
368
|
+
* Features:
|
|
369
|
+
* - Optimized for speed (10-100 results)
|
|
370
|
+
* - Supports advanced filtering and search algorithms
|
|
371
|
+
* - Rate limited to prevent quota exceeded errors
|
|
372
|
+
* - Returns structured ResultSet with metadata
|
|
373
|
+
*
|
|
374
|
+
* @param params - Search parameters including question, filters, and result count
|
|
375
|
+
* @returns Promise resolving to a ResultSet containing search results and metadata
|
|
376
|
+
*
|
|
377
|
+
* @example
|
|
378
|
+
* ```typescript
|
|
379
|
+
* // Basic fast search
|
|
380
|
+
* const results = await client.fastSearch({
|
|
381
|
+
* question: 'latest AI technology trends',
|
|
382
|
+
* nResults: 20
|
|
383
|
+
* });
|
|
384
|
+
*
|
|
385
|
+
* // Advanced fast search with filters
|
|
386
|
+
* const filteredResults = await client.fastSearch({
|
|
387
|
+
* question: 'machine learning startups',
|
|
388
|
+
* nResults: 50,
|
|
389
|
+
* algorithm: 'hybrid-2',
|
|
390
|
+
* minSimilarity: 0.7,
|
|
391
|
+
* mustInclude: ['artificial intelligence', 'funding'],
|
|
392
|
+
* mustExclude: ['cryptocurrency'],
|
|
393
|
+
* continent: 'North America'
|
|
394
|
+
* });
|
|
395
|
+
*
|
|
396
|
+
* // Process results
|
|
397
|
+
* console.log(`Found ${results.length} results`);
|
|
398
|
+
* for (const result of results) {
|
|
399
|
+
* console.log(`Title: ${result.title}`);
|
|
400
|
+
* console.log(`URL: ${result.url}`);
|
|
401
|
+
* console.log(`Similarity: ${result.similarity}`);
|
|
402
|
+
* }
|
|
403
|
+
* ```
|
|
404
|
+
*/
|
|
405
|
+
public async fastSearch(
|
|
406
|
+
params: ClientSearchParamsSchema
|
|
407
|
+
): Promise<ResultSet> {
|
|
408
|
+
let search: Search;
|
|
409
|
+
|
|
410
|
+
// Type guard to check if we have a Search instance
|
|
411
|
+
const hasSearchInstance = (
|
|
412
|
+
params: ClientSearchParamsSchema
|
|
413
|
+
): params is {search: Search} => {
|
|
414
|
+
return "search" in params;
|
|
415
|
+
};
|
|
416
|
+
|
|
417
|
+
if (hasSearchInstance(params)) {
|
|
418
|
+
// Use provided Search instance
|
|
419
|
+
if (params.search instanceof Search) {
|
|
420
|
+
search = params.search;
|
|
421
|
+
} else {
|
|
422
|
+
throw new Error("Invalid search parameter provided.");
|
|
423
|
+
}
|
|
424
|
+
} else {
|
|
425
|
+
// Create new Search instance from params
|
|
426
|
+
// Merge with default search parameters before creating Search instance
|
|
427
|
+
const mergedParams = this.mergeWithDefaultSearch(params);
|
|
428
|
+
search = new Search(mergedParams);
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Generate search body from Search instance */
|
|
432
|
+
const body = await search.searchBody({client: this});
|
|
433
|
+
|
|
434
|
+
/** Validate search body */
|
|
435
|
+
const validatedBody = validateFastSearchParams(body);
|
|
436
|
+
|
|
437
|
+
/** Make API request and parse response */
|
|
438
|
+
const {message, query, response} = (await this._request(
|
|
439
|
+
"fast-search",
|
|
440
|
+
validatedBody
|
|
441
|
+
)) as SearchResType;
|
|
442
|
+
|
|
443
|
+
const results = response.map((result) => {
|
|
444
|
+
const fullResult = {...query, ...result};
|
|
445
|
+
return fullResult;
|
|
446
|
+
});
|
|
447
|
+
/** Wrap response in ResultSet for enhanced functionality */
|
|
448
|
+
const resultSet = createResultSet(this, results);
|
|
449
|
+
return resultSet;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
public async fastSearches(
|
|
453
|
+
params: FastSearchesParamsType
|
|
454
|
+
): Promise<ResultSet[]> {
|
|
455
|
+
let searchSet: SearchSet;
|
|
456
|
+
|
|
457
|
+
// Handle SearchSet input
|
|
458
|
+
if (params instanceof SearchSet) {
|
|
459
|
+
searchSet = params;
|
|
460
|
+
} else {
|
|
461
|
+
// Handle questions array with shared search parameters
|
|
462
|
+
const {questions, ...sharedParams} = params;
|
|
463
|
+
|
|
464
|
+
// Merge shared parameters with default search parameters
|
|
465
|
+
const mergedSharedParams = this.mergeWithDefaultSearch(sharedParams);
|
|
466
|
+
|
|
467
|
+
// Create Search instances for each question using merged shared parameters
|
|
468
|
+
const searches = questions.map(
|
|
469
|
+
(question) => new Search({question, ...mergedSharedParams})
|
|
470
|
+
);
|
|
471
|
+
|
|
472
|
+
searchSet = new SearchSet(searches);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// Execute fastSearch for all searches in parallel
|
|
476
|
+
const results = await Promise.all(
|
|
477
|
+
searchSet.searches.map((search) => this.fastSearch({search}))
|
|
478
|
+
);
|
|
479
|
+
|
|
480
|
+
return results;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Performs an AI-powered search with custom instructions.
|
|
485
|
+
*
|
|
486
|
+
* AI search leverages advanced language models to understand complex queries
|
|
487
|
+
* and custom instructions, providing more contextually relevant results.
|
|
488
|
+
* This method is ideal for nuanced searches that require understanding intent,
|
|
489
|
+
* context, or specific analytical requirements.
|
|
490
|
+
*
|
|
491
|
+
* Features:
|
|
492
|
+
* - Natural language understanding of complex queries
|
|
493
|
+
* - Custom instructions for specialized search behavior
|
|
494
|
+
* - Context-aware result ranking and filtering
|
|
495
|
+
* - Rate limited for optimal performance
|
|
496
|
+
*
|
|
497
|
+
* @param params - AI search parameters including question and custom instructions
|
|
498
|
+
* @returns Promise resolving to a ResultSet containing AI-enhanced search results
|
|
499
|
+
*
|
|
500
|
+
* @example
|
|
501
|
+
* ```typescript
|
|
502
|
+
* // Basic AI search
|
|
503
|
+
* const results = await client.aiSearch({
|
|
504
|
+
* question: 'What are the environmental impacts of renewable energy?',
|
|
505
|
+
* instruction: 'Focus on recent studies and quantitative data'
|
|
506
|
+
* });
|
|
507
|
+
*
|
|
508
|
+
* // AI search with specific analytical focus
|
|
509
|
+
* const analysisResults = await client.aiSearch({
|
|
510
|
+
* question: 'company financial performance',
|
|
511
|
+
* instruction: 'Compare revenue growth and profit margins across quarters',
|
|
512
|
+
* nResults: 25,
|
|
513
|
+
* algorithm: 'hybrid-3'
|
|
514
|
+
* });
|
|
515
|
+
*
|
|
516
|
+
* // AI search for competitive analysis
|
|
517
|
+
* const competitiveResults = await client.aiSearch({
|
|
518
|
+
* question: 'competitors in the cloud computing market',
|
|
519
|
+
* instruction: 'Identify key players, market share, and competitive advantages'
|
|
520
|
+
* });
|
|
521
|
+
* ```
|
|
522
|
+
*/
|
|
523
|
+
public async aiSearch(params: UserAiSearchParamsType): Promise<ResultSet> {
|
|
524
|
+
/** Execute AI search within rate limiter */
|
|
525
|
+
/** Make direct API request with AI search parameters */
|
|
526
|
+
const json = (await this._request("search", params)) as SearchResType;
|
|
527
|
+
/** Wrap response in ResultSet for enhanced functionality */
|
|
528
|
+
|
|
529
|
+
const results = json.response.map((result) => {
|
|
530
|
+
const fullResult = {...json.query, ...result};
|
|
531
|
+
return fullResult;
|
|
532
|
+
});
|
|
533
|
+
const resultSet = createResultSet(this, results);
|
|
534
|
+
return resultSet;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Performs a bulk search for large-scale data collection.
|
|
539
|
+
*
|
|
540
|
+
* Bulk search is designed for comprehensive data gathering and analysis tasks.
|
|
541
|
+
* It supports large result sets (1000-10000 results) and provides efficient
|
|
542
|
+
* download mechanisms for extensive datasets. Results are encrypted and
|
|
543
|
+
* downloaded asynchronously for optimal performance.
|
|
544
|
+
*
|
|
545
|
+
* Features:
|
|
546
|
+
* - Large-scale data collection (1000-10000 results)
|
|
547
|
+
* - Encrypted result delivery for security
|
|
548
|
+
* - Asynchronous download processing
|
|
549
|
+
* - Comprehensive filtering and search options
|
|
550
|
+
* - Rate limited for optimal server performance
|
|
551
|
+
* - Supports both raw parameters and Search instances
|
|
552
|
+
*
|
|
553
|
+
* @param params - Bulk search parameters including question or search instance, with comprehensive filtering options
|
|
554
|
+
* @returns Promise resolving to a ResultSet containing bulk search results
|
|
555
|
+
*
|
|
556
|
+
* @example
|
|
557
|
+
* ```typescript
|
|
558
|
+
* // Basic bulk search
|
|
559
|
+
* const results = await client.bulkSearch({
|
|
560
|
+
* question: 'all articles about artificial intelligence',
|
|
561
|
+
* nResults: 5000
|
|
562
|
+
* });
|
|
563
|
+
*
|
|
564
|
+
* // Bulk search with comprehensive filtering
|
|
565
|
+
* const filteredResults = await client.bulkSearch({
|
|
566
|
+
* question: 'technology company funding rounds',
|
|
567
|
+
* nResults: 10000,
|
|
568
|
+
* algorithm: 'hybrid-3',
|
|
569
|
+
* minSimilarity: 0.6,
|
|
570
|
+
* continent: 'North America',
|
|
571
|
+
* region: 'Silicon Valley',
|
|
572
|
+
* mustInclude: ['venture capital', 'funding', 'investment'],
|
|
573
|
+
* mustExclude: ['cryptocurrency', 'NFT'],
|
|
574
|
+
* nProbes: 8,
|
|
575
|
+
* nContextify: 512
|
|
576
|
+
* });
|
|
577
|
+
*
|
|
578
|
+
* // Bulk search using a Search instance
|
|
579
|
+
* const search = new Search({
|
|
580
|
+
* question: 'renewable energy developments',
|
|
581
|
+
* nResults: 7500,
|
|
582
|
+
* algorithm: 'hybrid-2'
|
|
583
|
+
* });
|
|
584
|
+
* const searchResults = await client.bulkSearch({ search });
|
|
585
|
+
*
|
|
586
|
+
* // Process bulk results
|
|
587
|
+
* console.log(`Retrieved ${results.length} results`);
|
|
588
|
+
* const data = results.toJSON(); // Convert to JSON for analysis
|
|
589
|
+
* ```
|
|
590
|
+
*/
|
|
591
|
+
public async bulkSearch(
|
|
592
|
+
params: ClientSearchParamsSchema
|
|
593
|
+
): Promise<ResultSet> {
|
|
594
|
+
let search: Search;
|
|
595
|
+
|
|
596
|
+
// Type guard to check if we have a Search instance
|
|
597
|
+
const hasSearchInstance = (
|
|
598
|
+
params: ClientSearchParamsSchema
|
|
599
|
+
): params is {search: Search} => {
|
|
600
|
+
return "search" in params;
|
|
601
|
+
};
|
|
602
|
+
|
|
603
|
+
if (hasSearchInstance(params)) {
|
|
604
|
+
// Use provided Search instance
|
|
605
|
+
if (params.search instanceof Search) {
|
|
606
|
+
search = params.search;
|
|
607
|
+
} else {
|
|
608
|
+
throw new Error("Invalid search parameter provided.");
|
|
609
|
+
}
|
|
610
|
+
} else {
|
|
611
|
+
// Create new Search instance from params
|
|
612
|
+
// Merge with default search parameters before creating Search instance
|
|
613
|
+
const mergedParams = this.mergeWithDefaultSearch(params);
|
|
614
|
+
search = new Search(mergedParams);
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/** Generate search body from Search instance */
|
|
618
|
+
const body = await search.searchBody({client: this});
|
|
619
|
+
|
|
620
|
+
/** Validate search body for bulk search */
|
|
621
|
+
const validatedBody = validateBulkSearchParams(body);
|
|
622
|
+
|
|
623
|
+
/** Request bulk search and get encrypted file location */
|
|
624
|
+
const fileLocation = (await this._request(
|
|
625
|
+
"bulk-search",
|
|
626
|
+
validatedBody
|
|
627
|
+
)) as BulkResType;
|
|
628
|
+
|
|
629
|
+
/** Download and decrypt the bulk search results */
|
|
630
|
+
const json = (await bulkSearchDownload({
|
|
631
|
+
downloadFrom: fileLocation.download_from,
|
|
632
|
+
decryptUsing: fileLocation.decrypt_using,
|
|
633
|
+
})) as SearchResType;
|
|
634
|
+
const results = json.response.map((result) => {
|
|
635
|
+
const fullResult = {...json.query, ...result};
|
|
636
|
+
return fullResult;
|
|
637
|
+
});
|
|
638
|
+
/** Wrap response in ResultSet for enhanced functionality */
|
|
639
|
+
const resultSet = createResultSet(this, results);
|
|
640
|
+
return resultSet;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
/**
|
|
644
|
+
* Scrapes and extracts structured content from a web URL.
|
|
645
|
+
*
|
|
646
|
+
* This method fetches the content of a web page and extracts structured information
|
|
647
|
+
* including title, description, main content, metadata, and other relevant details.
|
|
648
|
+
* The scraping is optimized for clean content extraction and handles various
|
|
649
|
+
* website structures and formats.
|
|
650
|
+
*
|
|
651
|
+
* Features:
|
|
652
|
+
* - Clean content extraction from web pages
|
|
653
|
+
* - Structured data including metadata and content
|
|
654
|
+
* - Handles various website formats and structures
|
|
655
|
+
* - Rate limited to respect website policies
|
|
656
|
+
* - Returns WebPage object with enhanced functionality
|
|
657
|
+
*
|
|
658
|
+
* @param url - The URL to scrape and extract content from
|
|
659
|
+
* @returns Promise resolving to a WebPage object containing structured content
|
|
660
|
+
*
|
|
661
|
+
* @example
|
|
662
|
+
* ```typescript
|
|
663
|
+
* // Basic URL scraping
|
|
664
|
+
* const webpage = await client.scrapeUrl('https://example.com/article');
|
|
665
|
+
*
|
|
666
|
+
* // Access scraped content
|
|
667
|
+
* console.log(`Title: ${webpage.title}`);
|
|
668
|
+
* console.log(`Description: ${webpage.description}`);
|
|
669
|
+
* console.log(`Author: ${webpage.author}`);
|
|
670
|
+
* console.log(`Published: ${webpage.published}`);
|
|
671
|
+
* console.log(`Content length: ${webpage.content.length} characters`);
|
|
672
|
+
*
|
|
673
|
+
* // Extract specific content sections
|
|
674
|
+
* const mainContent = webpage.content;
|
|
675
|
+
* const bestChunk = webpage.bestChunk; // Most relevant content chunk
|
|
676
|
+
*
|
|
677
|
+
* // Use in combination with search
|
|
678
|
+
* const searchResults = await client.fastSearch({
|
|
679
|
+
* question: 'artificial intelligence news',
|
|
680
|
+
* nResults: 10
|
|
681
|
+
* });
|
|
682
|
+
*
|
|
683
|
+
* // Scrape the top result
|
|
684
|
+
* if (searchResults.length > 0) {
|
|
685
|
+
* const topArticle = await client.scrapeUrl(searchResults[0].url);
|
|
686
|
+
* console.log(`Full article: ${topArticle.content}`);
|
|
687
|
+
* }
|
|
688
|
+
* ```
|
|
689
|
+
*/
|
|
690
|
+
public async scrapeUrl(url: string): Promise<WebPageData> {
|
|
691
|
+
/** Execute URL scraping within rate limiter */
|
|
692
|
+
return this.limiters.scrapeUrl.schedule(async () => {
|
|
693
|
+
/** Make API request to scrape the URL */
|
|
694
|
+
const json = (await this._request("scrape-url", {url})) as ScrapeFullRes;
|
|
695
|
+
/** Wrap response in WebPage object for enhanced functionality */
|
|
696
|
+
const webPage = new WebPageData(this, json.response);
|
|
697
|
+
return webPage;
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Analyzes topic trends and popularity over time.
|
|
703
|
+
*
|
|
704
|
+
* This method provides insights into how a topic's popularity and relevance
|
|
705
|
+
* changes over time. It's useful for market research, content strategy,
|
|
706
|
+
* trend analysis, and understanding topic momentum.
|
|
707
|
+
*
|
|
708
|
+
* Features:
|
|
709
|
+
* - Historical trend analysis for any topic
|
|
710
|
+
* - Popularity metrics and temporal patterns
|
|
711
|
+
* - Optional SQL filtering for refined analysis
|
|
712
|
+
* - Returns TopicTrend object with analytical data
|
|
713
|
+
*
|
|
714
|
+
* @param query - The topic or query to analyze trends for
|
|
715
|
+
* @param sqlFilter - Optional SQL filter to refine the trend analysis
|
|
716
|
+
* @returns Promise resolving to a TopicTrend object containing trend analysis data
|
|
717
|
+
*
|
|
718
|
+
* @example
|
|
719
|
+
* ```typescript
|
|
720
|
+
* // Basic trend analysis
|
|
721
|
+
* const trends = await client.topicTrend('artificial intelligence');
|
|
722
|
+
*
|
|
723
|
+
* // Access trend data
|
|
724
|
+
* console.log(`Topic: ${trends.query}`);
|
|
725
|
+
* console.log(`Data points: ${trends.data.length}`);
|
|
726
|
+
* trends.data.forEach(point => {
|
|
727
|
+
* console.log(`Date: ${point.date}, Popularity: ${point.popularity}`);
|
|
728
|
+
* });
|
|
729
|
+
*
|
|
730
|
+
* // Trend analysis with filtering
|
|
731
|
+
* const filteredTrends = await client.topicTrend(
|
|
732
|
+
* 'machine learning',
|
|
733
|
+
* 'continent = "North America" AND published >= "2023-01-01"'
|
|
734
|
+
* );
|
|
735
|
+
*
|
|
736
|
+
* // Analyze multiple related topics
|
|
737
|
+
* const topics = ['AI', 'machine learning', 'deep learning', 'neural networks'];
|
|
738
|
+
* for (const topic of topics) {
|
|
739
|
+
* const trend = await client.topicTrend(topic);
|
|
740
|
+
* console.log(`${topic} trend analysis complete`);
|
|
741
|
+
* }
|
|
742
|
+
*
|
|
743
|
+
* // Use trend data for content strategy
|
|
744
|
+
* const trendingTopics = ['blockchain', 'quantum computing', 'renewable energy'];
|
|
745
|
+
* const trendAnalyses = await Promise.all(
|
|
746
|
+
* trendingTopics.map(topic => client.topicTrend(topic))
|
|
747
|
+
* );
|
|
748
|
+
* ```
|
|
749
|
+
*/
|
|
750
|
+
public async topicTrend(
|
|
751
|
+
query: string,
|
|
752
|
+
sqlFilter?: SqlFilter
|
|
753
|
+
): Promise<TopicTrend> {
|
|
754
|
+
/** Prepare request body with query and optional SQL filter */
|
|
755
|
+
const body = {query, sql_filter: sqlFilter};
|
|
756
|
+
/** Make API request for trend analysis */
|
|
757
|
+
const json = await this._request("topic-trend", body);
|
|
758
|
+
/** Wrap response in TopicTrend object for enhanced functionality */
|
|
759
|
+
const topicTrend = new TopicTrend(json);
|
|
760
|
+
return topicTrend;
|
|
761
|
+
}
|
|
762
|
+
}
|