nosible 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +438 -0
- package/dist/index.cjs +1841 -0
- package/dist/index.cjs.map +29 -0
- package/dist/index.js +1815 -0
- package/dist/index.js.map +29 -0
- package/package.json +63 -0
- package/src/api/api.test.ts +366 -0
- package/src/api/index.ts +179 -0
- package/src/api/schemas.ts +152 -0
- package/src/client.test.ts +685 -0
- package/src/client.ts +762 -0
- package/src/index.ts +4 -0
- package/src/scrape/types.ts +119 -0
- package/src/scrape/webPageData.test.ts +302 -0
- package/src/scrape/webPageData.ts +103 -0
- package/src/search/analyze.test.ts +396 -0
- package/src/search/analyze.ts +151 -0
- package/src/search/bulkSearch.ts +62 -0
- package/src/search/result.test.ts +423 -0
- package/src/search/result.ts +391 -0
- package/src/search/result.types.ts +32 -0
- package/src/search/resultFactory.ts +21 -0
- package/src/search/resultSet.io.test.ts +320 -0
- package/src/search/resultSet.test.ts +368 -0
- package/src/search/resultSet.ts +387 -0
- package/src/search/resultSet.types.ts +3 -0
- package/src/search/search.test.ts +299 -0
- package/src/search/search.ts +187 -0
- package/src/search/searchSet.io.test.ts +321 -0
- package/src/search/searchSet.ts +122 -0
- package/src/search/sqlFilter.test.ts +129 -0
- package/src/search/sqlFilter.ts +147 -0
- package/src/test-utils/mocks.ts +159 -0
- package/src/topicTrend/topicTrend.ts +53 -0
- package/src/utils/browser.test.ts +209 -0
- package/src/utils/browser.ts +21 -0
- package/src/utils/fernet.ts +47 -0
- package/src/utils/file.test.ts +81 -0
- package/src/utils/file.ts +195 -0
- package/src/utils/index.ts +7 -0
- package/src/utils/llm.test.ts +279 -0
- package/src/utils/llm.ts +244 -0
- package/src/utils/userPlan.test.ts +332 -0
- package/src/utils/userPlan.ts +211 -0
package/src/utils/llm.ts
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import OpenAI from "openai";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Default LLM model used for sentiment analysis and query expansion.
|
|
5
|
+
* Uses Google's Gemini 2.0 Flash model via OpenAI-compatible API.
|
|
6
|
+
*/
|
|
7
|
+
const defaultModel = "google/gemini-2.0-flash-001";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Analyzes the sentiment of a given text using an LLM.
|
|
11
|
+
*
|
|
12
|
+
* Returns a numeric score between -1.0 (very negative) and 1.0 (very positive).
|
|
13
|
+
* The function uses a structured prompt to ensure consistent numeric output.
|
|
14
|
+
*
|
|
15
|
+
* @param options - Configuration options for sentiment analysis
|
|
16
|
+
* @param options.text - The text to analyze for sentiment. Must not be empty or whitespace.
|
|
17
|
+
* @param options.model - The LLM model to use. Defaults to "google/gemini-2.0-flash-001".
|
|
18
|
+
* @param options.llmClient - OpenAI client instance for making API calls. Must be provided.
|
|
19
|
+
*
|
|
20
|
+
* @returns Promise<number> - A sentiment score between -1.0 and 1.0
|
|
21
|
+
*
|
|
22
|
+
* @throws {Error} When llmClient is not provided
|
|
23
|
+
* @throws {Error} When text is empty or contains only whitespace
|
|
24
|
+
* @throws {Error} When the LLM response is not a valid float
|
|
25
|
+
* @throws {Error} When the sentiment score is outside the valid range [-1.0, 1.0]
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```typescript
|
|
29
|
+
* import OpenAI from "openai";
|
|
30
|
+
* import { getSentiment } from "./llm";
|
|
31
|
+
*
|
|
32
|
+
* const client = new OpenAI({ apiKey: "your-api-key" });
|
|
33
|
+
* const score = await getSentiment({
|
|
34
|
+
* text: "I love this product!",
|
|
35
|
+
* llmClient: client
|
|
36
|
+
* });
|
|
37
|
+
* console.log(score); // Output: 0.8 (approximately)
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export const getSentiment = async ({
|
|
41
|
+
text,
|
|
42
|
+
model = defaultModel,
|
|
43
|
+
llmClient,
|
|
44
|
+
}: {
|
|
45
|
+
/** The text to analyze for sentiment */
|
|
46
|
+
text: string;
|
|
47
|
+
/** The LLM model to use for analysis */
|
|
48
|
+
model?: string;
|
|
49
|
+
/** OpenAI client instance for API calls */
|
|
50
|
+
llmClient: OpenAI | undefined;
|
|
51
|
+
}): Promise<number> => {
|
|
52
|
+
if (!llmClient) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
"A OpenAI client instance must be provided as 'llmClient'."
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (!text || text.trim() === "") {
|
|
59
|
+
throw new Error("Text must not be empty or whitespace.");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const prompt = `
|
|
63
|
+
# TASK DESCRIPTION
|
|
64
|
+
On a scale from -1.0 (very negative) to 1.0 (very positive),
|
|
65
|
+
please rate the sentiment of the following text and return _only_ the numeric score:
|
|
66
|
+
${text.trim()}
|
|
67
|
+
|
|
68
|
+
# RESPONSE FORMAT
|
|
69
|
+
|
|
70
|
+
The response must be a float in [-1.0, 1.0]. No other text must be returned.
|
|
71
|
+
`;
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
const completion = await llmClient.chat.completions.create({
|
|
75
|
+
model: model,
|
|
76
|
+
messages: [{role: "user", content: prompt.trim()}],
|
|
77
|
+
temperature: 0.7,
|
|
78
|
+
});
|
|
79
|
+
const raw = completion.choices[0]?.message?.content as string;
|
|
80
|
+
|
|
81
|
+
const score = parseFloat(raw);
|
|
82
|
+
if (Number.isNaN(score)) {
|
|
83
|
+
throw new Error(
|
|
84
|
+
`Sentiment response is not a float: ${JSON.stringify(raw)}`
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
if (score < -1.0 || score > 1.0) {
|
|
88
|
+
throw new Error(`Sentiment ${score} outside valid range [-1.0, 1.0]`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return score;
|
|
92
|
+
} catch (error) {
|
|
93
|
+
throw error;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Generates semantic query expansions to improve search recall.
|
|
99
|
+
*
|
|
100
|
+
* Creates 10 semantically equivalent but lexically different questions
|
|
101
|
+
* that maintain the same intent while using different wording and entities.
|
|
102
|
+
* This helps search engines find more relevant documents through query variation.
|
|
103
|
+
*
|
|
104
|
+
* The generated expansions follow specific guidelines:
|
|
105
|
+
* - Include specific named entities (people, places, organizations)
|
|
106
|
+
* - Are highly targeted and semantically unambiguous
|
|
107
|
+
* - Use between 10-15 words per question
|
|
108
|
+
* - Leverage semantic and contextual expansion techniques
|
|
109
|
+
*
|
|
110
|
+
* @param options - Configuration options for query expansion
|
|
111
|
+
* @param options.question - The original search question to expand. Must not be empty or whitespace.
|
|
112
|
+
* @param options.model - The LLM model to use. Defaults to "google/gemini-2.0-flash-001".
|
|
113
|
+
* @param options.llmClient - OpenAI client instance for making API calls. Must be provided.
|
|
114
|
+
*
|
|
115
|
+
* @returns Promise<string[]> - An array of 10 expanded query strings
|
|
116
|
+
*
|
|
117
|
+
* @throws {Error} When llmClient is not provided
|
|
118
|
+
* @throws {Error} When question is empty or contains only whitespace
|
|
119
|
+
* @throws {Error} When the LLM response is not valid JSON
|
|
120
|
+
* @throws {Error} When the response doesn't contain exactly 10 string items
|
|
121
|
+
*
|
|
122
|
+
* @example
|
|
123
|
+
* ```typescript
|
|
124
|
+
* import OpenAI from "openai";
|
|
125
|
+
* import { generateExpansions } from "./llm";
|
|
126
|
+
*
|
|
127
|
+
* const client = new OpenAI({ apiKey: "your-api-key" });
|
|
128
|
+
* const expansions = await generateExpansions({
|
|
129
|
+
* question: "best restaurants in New York",
|
|
130
|
+
* llmClient: client
|
|
131
|
+
* });
|
|
132
|
+
* console.log(expansions);
|
|
133
|
+
* // Output: [
|
|
134
|
+
* // "top dining establishments Manhattan NYC",
|
|
135
|
+
* // "fine eating places New York City",
|
|
136
|
+
* // "recommended food venues NYC boroughs",
|
|
137
|
+
* // ...
|
|
138
|
+
* // ]
|
|
139
|
+
* ```
|
|
140
|
+
*/
|
|
141
|
+
export const generateExpansions = async ({
|
|
142
|
+
question,
|
|
143
|
+
model = defaultModel,
|
|
144
|
+
llmClient,
|
|
145
|
+
}: {
|
|
146
|
+
/** The original search question to expand */
|
|
147
|
+
question: string;
|
|
148
|
+
/** The LLM model to use for expansion */
|
|
149
|
+
model?: string;
|
|
150
|
+
/** OpenAI client instance for API calls */
|
|
151
|
+
llmClient: OpenAI | undefined;
|
|
152
|
+
}): Promise<string[]> => {
|
|
153
|
+
if (!llmClient) {
|
|
154
|
+
throw new Error(
|
|
155
|
+
"A OpenAI client instance must be provided as 'llmClient'."
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (!question || question.trim() === "") {
|
|
160
|
+
throw new Error("Question must not be empty or whitespace.");
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const prompt = `
|
|
164
|
+
# TASK DESCRIPTION
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
Given a search question you must generate a list of 10 similar questions that have the same exact
|
|
168
|
+
semantic meaning but are contextually and lexically different to improve search recall.
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
## Question
|
|
172
|
+
|
|
173
|
+
Here is the question you must generate expansions for:
|
|
174
|
+
|
|
175
|
+
Question: ${question.trim()}
|
|
176
|
+
|
|
177
|
+
# RESPONSE FORMAT
|
|
178
|
+
|
|
179
|
+
Your response must be a JSON object structured as follows: a list of ten strings. Each string must
|
|
180
|
+
be a grammatically correct question that expands on the original question to improve recall.
|
|
181
|
+
|
|
182
|
+
[
|
|
183
|
+
string,
|
|
184
|
+
string,
|
|
185
|
+
string,
|
|
186
|
+
string,
|
|
187
|
+
string,
|
|
188
|
+
string,
|
|
189
|
+
string,
|
|
190
|
+
string,
|
|
191
|
+
string,
|
|
192
|
+
string
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
# EXPANSION GUIDELINES
|
|
196
|
+
|
|
197
|
+
1. **Use specific named entities** - To improve the quality of your search results you must mention
|
|
198
|
+
specific named entities (people, locations, organizations, products, places) in expansions.
|
|
199
|
+
2. **Expansions must be highly targeted** - To improve the quality of search results each expansion
|
|
200
|
+
must be semantically unambiguous. Questions must be use between ten and fifteen words.
|
|
201
|
+
3. **Expansions must improve recall** - When expanding the question leverage semantic and contextual
|
|
202
|
+
expansion to maximize the ability of the search engine to find semantically relevant documents:
|
|
203
|
+
- Semantic Example: Swap "climate change" with "global warming" or "environmental change".
|
|
204
|
+
- Contextual Example: Swap "diabetes treatment" with "insulin therapy" or "blood sugar management".
|
|
205
|
+
`;
|
|
206
|
+
|
|
207
|
+
try {
|
|
208
|
+
const completion = await llmClient.chat.completions.create({
|
|
209
|
+
model: model,
|
|
210
|
+
messages: [{role: "user", content: prompt.trim()}],
|
|
211
|
+
temperature: 0.7,
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
let raw = (completion.choices[0]?.message?.content as string) ?? "";
|
|
215
|
+
let toParse = raw.trim();
|
|
216
|
+
|
|
217
|
+
if (toParse.startsWith("```")) {
|
|
218
|
+
toParse = toParse.replace(/^```+/, "").trim();
|
|
219
|
+
if (toParse.toLowerCase().startsWith("json")) {
|
|
220
|
+
toParse = toParse.slice(4).trim();
|
|
221
|
+
}
|
|
222
|
+
toParse = toParse.replace(/```+$/, "").trim();
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
let expansions: unknown;
|
|
226
|
+
try {
|
|
227
|
+
expansions = JSON.parse(toParse);
|
|
228
|
+
} catch (decodeErr) {
|
|
229
|
+
throw new Error(`OpenRouter response was not valid JSON: '${toParse}'`);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
if (
|
|
233
|
+
!Array.isArray(expansions) ||
|
|
234
|
+
expansions.length !== 10 ||
|
|
235
|
+
!expansions.every((q) => typeof q === "string")
|
|
236
|
+
) {
|
|
237
|
+
throw new Error("Invalid response: 'choices' missing or empty");
|
|
238
|
+
}
|
|
239
|
+
console.debug("Expansions: ", expansions);
|
|
240
|
+
return expansions as string[];
|
|
241
|
+
} catch (error) {
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
};
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { getUserPlan, getLimiters, type Plan } from "./userPlan";
|
|
3
|
+
import Bottleneck from "bottleneck";
|
|
4
|
+
|
|
5
|
+
describe("getUserPlan", () => {
|
|
6
|
+
test("returns correct plan for test API key", () => {
|
|
7
|
+
const plan = getUserPlan("test|some-api-key");
|
|
8
|
+
expect(plan.key).toBe("test");
|
|
9
|
+
expect(plan.name).toBe("Test");
|
|
10
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
11
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(300);
|
|
12
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
13
|
+
expect(plan.rateLimits.bulk.month).toBe(300);
|
|
14
|
+
expect(plan.rateLimits.fast.minute).toBe(60);
|
|
15
|
+
expect(plan.rateLimits.fast.month).toBe(3000);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("returns correct plan for basic API key", () => {
|
|
19
|
+
const plan = getUserPlan("basic|some-api-key");
|
|
20
|
+
expect(plan.key).toBe("basic");
|
|
21
|
+
expect(plan.name).toBe("Basic");
|
|
22
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
23
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(1400);
|
|
24
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
25
|
+
expect(plan.rateLimits.bulk.month).toBe(1400);
|
|
26
|
+
expect(plan.rateLimits.fast.minute).toBe(60);
|
|
27
|
+
expect(plan.rateLimits.fast.month).toBe(14000);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("returns correct plan for pro API key", () => {
|
|
31
|
+
const plan = getUserPlan("pro|some-api-key");
|
|
32
|
+
expect(plan.key).toBe("pro");
|
|
33
|
+
expect(plan.name).toBe("Pro");
|
|
34
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
35
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(6700);
|
|
36
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
37
|
+
expect(plan.rateLimits.bulk.month).toBe(6700);
|
|
38
|
+
expect(plan.rateLimits.fast.minute).toBe(60);
|
|
39
|
+
expect(plan.rateLimits.fast.month).toBe(67000);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("returns correct plan for pro+ API key", () => {
|
|
43
|
+
const plan = getUserPlan("pro+|some-api-key");
|
|
44
|
+
expect(plan.key).toBe("pro+");
|
|
45
|
+
expect(plan.name).toBe("Pro+");
|
|
46
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
47
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(32000);
|
|
48
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
49
|
+
expect(plan.rateLimits.bulk.month).toBe(32000);
|
|
50
|
+
expect(plan.rateLimits.fast.minute).toBe(60);
|
|
51
|
+
expect(plan.rateLimits.fast.month).toBe(320000);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("returns correct plan for business API key", () => {
|
|
55
|
+
const plan = getUserPlan("bus|some-api-key");
|
|
56
|
+
expect(plan.key).toBe("bus");
|
|
57
|
+
expect(plan.name).toBe("Business");
|
|
58
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
59
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(200000);
|
|
60
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
61
|
+
expect(plan.rateLimits.bulk.month).toBe(200000);
|
|
62
|
+
expect(plan.rateLimits.fast.minute).toBe(60);
|
|
63
|
+
expect(plan.rateLimits.fast.month).toBe(2000000);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("returns correct plan for business+ API key", () => {
|
|
67
|
+
const plan = getUserPlan("bus+|some-api-key");
|
|
68
|
+
expect(plan.key).toBe("bus+");
|
|
69
|
+
expect(plan.name).toBe("Business+");
|
|
70
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
71
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(500000);
|
|
72
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
73
|
+
expect(plan.rateLimits.bulk.month).toBe(500000);
|
|
74
|
+
expect(plan.rateLimits.fast.minute).toBe(120);
|
|
75
|
+
expect(plan.rateLimits.fast.month).toBe(5000000);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("returns correct plan for enterprise API key", () => {
|
|
79
|
+
const plan = getUserPlan("ent|some-api-key");
|
|
80
|
+
expect(plan.key).toBe("ent");
|
|
81
|
+
expect(plan.name).toBe("Enterprise");
|
|
82
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
83
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(1500000);
|
|
84
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
85
|
+
expect(plan.rateLimits.bulk.month).toBe(1500000);
|
|
86
|
+
expect(plan.rateLimits.fast.minute).toBe(360);
|
|
87
|
+
expect(plan.rateLimits.fast.month).toBe(15000000);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("returns correct plan for production API key", () => {
|
|
91
|
+
const plan = getUserPlan("prod|some-api-key");
|
|
92
|
+
expect(plan.key).toBe("prod");
|
|
93
|
+
expect(plan.name).toBe("Production");
|
|
94
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
95
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(1500000);
|
|
96
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
97
|
+
expect(plan.rateLimits.bulk.month).toBe(1500000);
|
|
98
|
+
expect(plan.rateLimits.fast.minute).toBe(360);
|
|
99
|
+
expect(plan.rateLimits.fast.month).toBe(15000000);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("returns correct plan for chat API key", () => {
|
|
103
|
+
const plan = getUserPlan("chat|some-api-key");
|
|
104
|
+
expect(plan.key).toBe("chat");
|
|
105
|
+
expect(plan.name).toBe("Chat");
|
|
106
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
107
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(1500000);
|
|
108
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
109
|
+
expect(plan.rateLimits.bulk.month).toBe(1500000);
|
|
110
|
+
expect(plan.rateLimits.fast.minute).toBe(360);
|
|
111
|
+
expect(plan.rateLimits.fast.month).toBe(15000000);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("returns correct plan for self API key", () => {
|
|
115
|
+
const plan = getUserPlan("self|some-api-key");
|
|
116
|
+
expect(plan.key).toBe("self");
|
|
117
|
+
expect(plan.name).toBe("Self");
|
|
118
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(6000);
|
|
119
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(1500000);
|
|
120
|
+
expect(plan.rateLimits.bulk.minute).toBe(6000);
|
|
121
|
+
expect(plan.rateLimits.bulk.month).toBe(1500000);
|
|
122
|
+
expect(plan.rateLimits.fast.minute).toBe(36000);
|
|
123
|
+
expect(plan.rateLimits.fast.month).toBe(15000000);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("returns correct plan for consumer API key", () => {
|
|
127
|
+
const plan = getUserPlan("cons|some-api-key");
|
|
128
|
+
expect(plan.key).toBe("cons");
|
|
129
|
+
expect(plan.name).toBe("Consumer");
|
|
130
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
131
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(3000);
|
|
132
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
133
|
+
expect(plan.rateLimits.bulk.month).toBe(3000);
|
|
134
|
+
expect(plan.rateLimits.fast.minute).toBe(120);
|
|
135
|
+
expect(plan.rateLimits.fast.month).toBe(30000);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("returns correct plan for startup API key", () => {
|
|
139
|
+
const plan = getUserPlan("stup|some-api-key");
|
|
140
|
+
expect(plan.key).toBe("stup");
|
|
141
|
+
expect(plan.name).toBe("Startup");
|
|
142
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
143
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(30000);
|
|
144
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
145
|
+
expect(plan.rateLimits.bulk.month).toBe(30000);
|
|
146
|
+
expect(plan.rateLimits.fast.minute).toBe(360);
|
|
147
|
+
expect(plan.rateLimits.fast.month).toBe(300000);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("returns correct plan for business network API key", () => {
|
|
151
|
+
const plan = getUserPlan("busn|some-api-key");
|
|
152
|
+
expect(plan.key).toBe("busn");
|
|
153
|
+
expect(plan.name).toBe("Business Network");
|
|
154
|
+
expect(plan.rateLimits.scrapeUrl.minute).toBe(60);
|
|
155
|
+
expect(plan.rateLimits.scrapeUrl.month).toBe(300000);
|
|
156
|
+
expect(plan.rateLimits.bulk.minute).toBe(60);
|
|
157
|
+
expect(plan.rateLimits.bulk.month).toBe(300000);
|
|
158
|
+
expect(plan.rateLimits.fast.minute).toBe(360);
|
|
159
|
+
expect(plan.rateLimits.fast.month).toBe(3000000);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
test("handles API key with additional pipe characters", () => {
|
|
163
|
+
const plan = getUserPlan("pro|key|with|multiple|pipes");
|
|
164
|
+
expect(plan.key).toBe("pro");
|
|
165
|
+
expect(plan.name).toBe("Pro");
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
test("throws error for invalid API key", () => {
|
|
169
|
+
expect(() => getUserPlan("invalid|some-api-key")).toThrow("Invalid API key");
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("throws error for API key with unknown plan", () => {
|
|
173
|
+
expect(() => getUserPlan("unknown|some-api-key")).toThrow("Invalid API key");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("throws error for empty API key", () => {
|
|
177
|
+
expect(() => getUserPlan("")).toThrow("Invalid API key");
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("throws error for API key with only pipe", () => {
|
|
181
|
+
expect(() => getUserPlan("|some-api-key")).toThrow("Invalid API key");
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe("getLimiters", () => {
|
|
186
|
+
test("returns Bottleneck limiters for test plan", () => {
|
|
187
|
+
const limiters = getLimiters("test|some-api-key");
|
|
188
|
+
|
|
189
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
190
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
191
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
test("returns Bottleneck limiters for pro plan", () => {
|
|
195
|
+
const limiters = getLimiters("pro|some-api-key");
|
|
196
|
+
|
|
197
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
198
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
199
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("returns Bottleneck limiters for enterprise plan", () => {
|
|
203
|
+
const limiters = getLimiters("ent|some-api-key");
|
|
204
|
+
|
|
205
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
206
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
207
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
test("returns Bottleneck limiters for self plan", () => {
|
|
211
|
+
const limiters = getLimiters("self|some-api-key");
|
|
212
|
+
|
|
213
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
214
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
215
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
test("limiters have correct reservoir settings for basic plan", () => {
|
|
219
|
+
const limiters = getLimiters("basic|some-api-key");
|
|
220
|
+
|
|
221
|
+
// Test plan limits: scrapeUrl: {minute: 60, month: 1400}
|
|
222
|
+
const scrapeUrlLimiter = limiters.scrapeUrl;
|
|
223
|
+
expect(scrapeUrlLimiter).toBeInstanceOf(Bottleneck);
|
|
224
|
+
|
|
225
|
+
// Test plan limits: bulk: {minute: 60, month: 1400}
|
|
226
|
+
const bulkLimiter = limiters.bulk;
|
|
227
|
+
expect(bulkLimiter).toBeInstanceOf(Bottleneck);
|
|
228
|
+
|
|
229
|
+
// Test plan limits: fast: {minute: 60, month: 14000}
|
|
230
|
+
const fastLimiter = limiters.fast;
|
|
231
|
+
expect(fastLimiter).toBeInstanceOf(Bottleneck);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
test("limiters have different settings for business+ plan", () => {
|
|
235
|
+
const limiters = getLimiters("bus+|some-api-key");
|
|
236
|
+
|
|
237
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
238
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
239
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
test("limiters have different settings for enterprise plan", () => {
|
|
243
|
+
const limiters = getLimiters("ent|some-api-key");
|
|
244
|
+
|
|
245
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
246
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
247
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test("limiters have different settings for self plan", () => {
|
|
251
|
+
const limiters = getLimiters("self|some-api-key");
|
|
252
|
+
|
|
253
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
254
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
255
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
test("throws error for invalid plan key", () => {
|
|
259
|
+
expect(() => getLimiters("invalid|some-api-key")).toThrow("Invalid API key");
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
test("returns separate limiter instances", () => {
|
|
263
|
+
const limiters1 = getLimiters("test|some-api-key");
|
|
264
|
+
const limiters2 = getLimiters("test|some-api-key");
|
|
265
|
+
|
|
266
|
+
expect(limiters1.scrapeUrl).not.toBe(limiters2.scrapeUrl);
|
|
267
|
+
expect(limiters1.bulk).not.toBe(limiters2.bulk);
|
|
268
|
+
expect(limiters1.fast).not.toBe(limiters2.fast);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("limiters are chained (minute + month)", () => {
|
|
272
|
+
const limiters = getLimiters("test|some-api-key");
|
|
273
|
+
|
|
274
|
+
// All limiters should be chained limiters (minute + month)
|
|
275
|
+
expect(limiters.scrapeUrl).toBeInstanceOf(Bottleneck);
|
|
276
|
+
expect(limiters.bulk).toBeInstanceOf(Bottleneck);
|
|
277
|
+
expect(limiters.fast).toBeInstanceOf(Bottleneck);
|
|
278
|
+
});
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
describe("Plan type validation", () => {
|
|
282
|
+
test("all plans have required properties", () => {
|
|
283
|
+
const plans: Plan[] = [
|
|
284
|
+
{ key: "test", name: "Test", rateLimits: { scrapeUrl: { minute: 60, month: 300 }, bulk: { minute: 60, month: 300 }, fast: { minute: 60, month: 3000 } } },
|
|
285
|
+
{ key: "basic", name: "Basic", rateLimits: { scrapeUrl: { minute: 60, month: 1400 }, bulk: { minute: 60, month: 1400 }, fast: { minute: 60, month: 14000 } } },
|
|
286
|
+
{ key: "pro", name: "Pro", rateLimits: { scrapeUrl: { minute: 60, month: 6700 }, bulk: { minute: 60, month: 6700 }, fast: { minute: 60, month: 67000 } } },
|
|
287
|
+
];
|
|
288
|
+
|
|
289
|
+
plans.forEach(plan => {
|
|
290
|
+
expect(plan).toHaveProperty("key");
|
|
291
|
+
expect(plan).toHaveProperty("name");
|
|
292
|
+
expect(plan).toHaveProperty("rateLimits");
|
|
293
|
+
expect(plan.rateLimits).toHaveProperty("scrapeUrl");
|
|
294
|
+
expect(plan.rateLimits).toHaveProperty("bulk");
|
|
295
|
+
expect(plan.rateLimits).toHaveProperty("fast");
|
|
296
|
+
|
|
297
|
+
expect(plan.rateLimits.scrapeUrl).toHaveProperty("minute");
|
|
298
|
+
expect(plan.rateLimits.scrapeUrl).toHaveProperty("month");
|
|
299
|
+
expect(plan.rateLimits.bulk).toHaveProperty("minute");
|
|
300
|
+
expect(plan.rateLimits.bulk).toHaveProperty("month");
|
|
301
|
+
expect(plan.rateLimits.fast).toHaveProperty("minute");
|
|
302
|
+
expect(plan.rateLimits.fast).toHaveProperty("month");
|
|
303
|
+
|
|
304
|
+
expect(typeof plan.rateLimits.scrapeUrl.minute).toBe("number");
|
|
305
|
+
expect(typeof plan.rateLimits.scrapeUrl.month).toBe("number");
|
|
306
|
+
expect(typeof plan.rateLimits.bulk.minute).toBe("number");
|
|
307
|
+
expect(typeof plan.rateLimits.bulk.month).toBe("number");
|
|
308
|
+
expect(typeof plan.rateLimits.fast.minute).toBe("number");
|
|
309
|
+
expect(typeof plan.rateLimits.fast.month).toBe("number");
|
|
310
|
+
});
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
test("plan limits are progressive", () => {
|
|
314
|
+
const testPlan = getUserPlan("test|key");
|
|
315
|
+
const basicPlan = getUserPlan("basic|key");
|
|
316
|
+
const proPlan = getUserPlan("pro|key");
|
|
317
|
+
const busPlan = getUserPlan("bus|key");
|
|
318
|
+
|
|
319
|
+
// Monthly limits should increase with plan tier
|
|
320
|
+
expect(testPlan.rateLimits.scrapeUrl.month).toBeLessThan(basicPlan.rateLimits.scrapeUrl.month);
|
|
321
|
+
expect(basicPlan.rateLimits.scrapeUrl.month).toBeLessThan(proPlan.rateLimits.scrapeUrl.month);
|
|
322
|
+
expect(proPlan.rateLimits.scrapeUrl.month).toBeLessThan(busPlan.rateLimits.scrapeUrl.month);
|
|
323
|
+
|
|
324
|
+
expect(testPlan.rateLimits.bulk.month).toBeLessThan(basicPlan.rateLimits.bulk.month);
|
|
325
|
+
expect(basicPlan.rateLimits.bulk.month).toBeLessThan(proPlan.rateLimits.bulk.month);
|
|
326
|
+
expect(proPlan.rateLimits.bulk.month).toBeLessThan(busPlan.rateLimits.bulk.month);
|
|
327
|
+
|
|
328
|
+
expect(testPlan.rateLimits.fast.month).toBeLessThan(basicPlan.rateLimits.fast.month);
|
|
329
|
+
expect(basicPlan.rateLimits.fast.month).toBeLessThan(proPlan.rateLimits.fast.month);
|
|
330
|
+
expect(proPlan.rateLimits.fast.month).toBeLessThan(busPlan.rateLimits.fast.month);
|
|
331
|
+
});
|
|
332
|
+
});
|