@shaivpidadi/trends-js 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -106
- package/dist/cjs/constants.js +4 -4
- package/dist/cjs/helpers/googleTrendsAPI.d.ts +3 -3
- package/dist/cjs/helpers/googleTrendsAPI.js +61 -57
- package/dist/cjs/helpers/request.js +21 -4
- package/dist/cjs/helpers/session-manager.d.ts +36 -0
- package/dist/cjs/helpers/session-manager.js +172 -0
- package/dist/cjs/index.d.ts +4 -4
- package/dist/esm/constants.js +4 -4
- package/dist/esm/helpers/googleTrendsAPI.d.ts +3 -3
- package/dist/esm/helpers/googleTrendsAPI.js +61 -57
- package/dist/esm/helpers/request.js +21 -4
- package/dist/esm/helpers/session-manager.d.ts +36 -0
- package/dist/esm/helpers/session-manager.js +165 -0
- package/dist/esm/index.d.ts +4 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -22,7 +22,6 @@ npm install @shaivpidadi/trends-js
|
|
|
22
22
|
- Get interest by region data
|
|
23
23
|
- Get related topics for any keyword
|
|
24
24
|
- Get related queries for any keyword
|
|
25
|
-
- Get combined related data (topics + queries)
|
|
26
25
|
- TypeScript support
|
|
27
26
|
- Promise-based API
|
|
28
27
|
|
|
@@ -113,8 +112,6 @@ const suggestions = await GoogleTrendsApi.autocomplete(
|
|
|
113
112
|
'bitcoin', // Keyword to get suggestions for
|
|
114
113
|
'en-US', // Language (default: 'en-US')
|
|
115
114
|
);
|
|
116
|
-
|
|
117
|
-
// Returns: string[]
|
|
118
115
|
```
|
|
119
116
|
|
|
120
117
|
### Explore
|
|
@@ -124,8 +121,8 @@ Get widget data for a keyword:
|
|
|
124
121
|
```typescript
|
|
125
122
|
const result = await GoogleTrendsApi.explore({
|
|
126
123
|
keyword: 'bitcoin',
|
|
127
|
-
geo: 'US'
|
|
128
|
-
time: 'today 12-m',
|
|
124
|
+
geo: 'US' // Default: 'US',
|
|
125
|
+
time: 'today 12-m',
|
|
129
126
|
category: 0, // Default: 0
|
|
130
127
|
property: '', // Default: ''
|
|
131
128
|
hl: 'en-US', // Default: 'en-US'
|
|
@@ -140,6 +137,7 @@ const result = await GoogleTrendsApi.explore({
|
|
|
140
137
|
// }>
|
|
141
138
|
// }
|
|
142
139
|
```
|
|
140
|
+
> **Note:** For all methods below, it is recommended to set `enableBackoff: true` in the options to automatically handle and bypass Google's rate limiting.
|
|
143
141
|
|
|
144
142
|
### Interest by Region
|
|
145
143
|
|
|
@@ -177,31 +175,20 @@ const result = await GoogleTrendsApi.interestByRegion({
|
|
|
177
175
|
// }
|
|
178
176
|
```
|
|
179
177
|
|
|
180
|
-
Example with multiple keywords and regions:
|
|
181
|
-
|
|
182
|
-
```typescript
|
|
183
|
-
const result = await GoogleTrendsApi.interestByRegion({
|
|
184
|
-
keyword: ['wine', 'peanuts'],
|
|
185
|
-
geo: ['US-CA', 'US-VA'],
|
|
186
|
-
startTime: new Date('2024-01-01'),
|
|
187
|
-
endTime: new Date(),
|
|
188
|
-
resolution: 'CITY',
|
|
189
|
-
enableBackoff: true
|
|
190
|
-
});
|
|
191
|
-
```
|
|
192
|
-
|
|
193
178
|
### Related Topics
|
|
194
179
|
|
|
195
180
|
Get related topics for any keyword:
|
|
196
181
|
|
|
197
182
|
```typescript
|
|
198
183
|
const result = await GoogleTrendsApi.relatedTopics({
|
|
199
|
-
keyword: 'artificial intelligence',
|
|
200
|
-
|
|
201
|
-
|
|
184
|
+
keyword: 'artificial intelligence',
|
|
185
|
+
startTime: new Date('2024-01-01'),
|
|
186
|
+
endTime: new Date(),
|
|
202
187
|
category: 0, // Optional - defaults to 0
|
|
188
|
+
geo: 'US', // Optional - defaults to 'US'
|
|
203
189
|
property: '', // Optional - defaults to ''
|
|
204
190
|
hl: 'en-US', // Optional - defaults to 'en-US'
|
|
191
|
+
enableBackoff: true // Optional - defaults to false
|
|
205
192
|
});
|
|
206
193
|
|
|
207
194
|
// Result structure:
|
|
@@ -232,12 +219,13 @@ Get related queries for any keyword:
|
|
|
232
219
|
|
|
233
220
|
```typescript
|
|
234
221
|
const result = await GoogleTrendsApi.relatedQueries({
|
|
235
|
-
keyword: 'machine learning',
|
|
236
|
-
geo: 'US',
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
hl: 'en-US',
|
|
222
|
+
keyword: 'machine learning',
|
|
223
|
+
geo: 'US',
|
|
224
|
+
startTime: new Date('2024-01-01'),
|
|
225
|
+
endTime: new Date(),
|
|
226
|
+
category: 0,
|
|
227
|
+
hl: 'en-US',
|
|
228
|
+
enableBackoff: true // Optional - defaults to false
|
|
241
229
|
});
|
|
242
230
|
|
|
243
231
|
// Result structure:
|
|
@@ -258,37 +246,14 @@ const result = await GoogleTrendsApi.relatedQueries({
|
|
|
258
246
|
// }
|
|
259
247
|
```
|
|
260
248
|
|
|
261
|
-
### Combined Related Data
|
|
262
|
-
|
|
263
|
-
Get both related topics and queries in a single call:
|
|
264
|
-
|
|
265
|
-
```typescript
|
|
266
|
-
const result = await GoogleTrendsApi.relatedData({
|
|
267
|
-
keyword: 'blockchain', // Required
|
|
268
|
-
geo: 'US', // Optional - defaults to 'US'
|
|
269
|
-
time: 'now 1-d', // Optional - defaults to 'now 1-d'
|
|
270
|
-
category: 0, // Optional - defaults to 0
|
|
271
|
-
property: '', // Optional - defaults to ''
|
|
272
|
-
hl: 'en-US', // Optional - defaults to 'en-US'
|
|
273
|
-
});
|
|
274
|
-
|
|
275
|
-
// Result structure:
|
|
276
|
-
// {
|
|
277
|
-
// data: {
|
|
278
|
-
// topics: Array<RelatedTopic>,
|
|
279
|
-
// queries: Array<RelatedQuery>
|
|
280
|
-
// }
|
|
281
|
-
// }
|
|
282
|
-
```
|
|
283
|
-
|
|
284
249
|
## API Reference
|
|
285
250
|
|
|
286
251
|
### DailyTrendsOptions
|
|
287
252
|
|
|
288
253
|
```typescript
|
|
289
254
|
interface DailyTrendsOptions {
|
|
290
|
-
geo?: string;
|
|
291
|
-
lang?: string;
|
|
255
|
+
geo?: string;
|
|
256
|
+
lang?: string;
|
|
292
257
|
}
|
|
293
258
|
```
|
|
294
259
|
|
|
@@ -297,7 +262,7 @@ interface DailyTrendsOptions {
|
|
|
297
262
|
```typescript
|
|
298
263
|
interface RealTimeTrendsOptions {
|
|
299
264
|
geo: string;
|
|
300
|
-
trendingHours?: number;
|
|
265
|
+
trendingHours?: number;
|
|
301
266
|
}
|
|
302
267
|
```
|
|
303
268
|
|
|
@@ -306,11 +271,11 @@ interface RealTimeTrendsOptions {
|
|
|
306
271
|
```typescript
|
|
307
272
|
interface ExploreOptions {
|
|
308
273
|
keyword: string;
|
|
309
|
-
geo?: string;
|
|
310
|
-
time?: string;
|
|
311
|
-
category?: number;
|
|
312
|
-
property?: string;
|
|
313
|
-
hl?: string;
|
|
274
|
+
geo?: string;
|
|
275
|
+
time?: string;
|
|
276
|
+
category?: number;
|
|
277
|
+
property?: string;
|
|
278
|
+
hl?: string;
|
|
314
279
|
}
|
|
315
280
|
```
|
|
316
281
|
|
|
@@ -318,64 +283,45 @@ interface ExploreOptions {
|
|
|
318
283
|
|
|
319
284
|
```typescript
|
|
320
285
|
interface InterestByRegionOptions {
|
|
321
|
-
keyword: string
|
|
322
|
-
startTime?: Date;
|
|
323
|
-
endTime?: Date;
|
|
324
|
-
geo?: string
|
|
325
|
-
resolution?: 'COUNTRY' | 'REGION' | 'CITY' | 'DMA';
|
|
326
|
-
hl?: string;
|
|
327
|
-
timezone?: number;
|
|
328
|
-
category?: number;
|
|
329
|
-
enableBackoff?: boolean
|
|
330
|
-
}
|
|
331
|
-
```
|
|
332
|
-
|
|
333
|
-
### RelatedTopicsResponse
|
|
334
|
-
|
|
335
|
-
```typescript
|
|
336
|
-
interface RelatedTopicsResponse {
|
|
337
|
-
default: {
|
|
338
|
-
rankedList: Array<{
|
|
339
|
-
rankedKeyword: Array<{
|
|
340
|
-
topic: {
|
|
341
|
-
mid: string;
|
|
342
|
-
title: string;
|
|
343
|
-
type: string;
|
|
344
|
-
};
|
|
345
|
-
value: number;
|
|
346
|
-
formattedValue: string;
|
|
347
|
-
hasData: boolean;
|
|
348
|
-
link: string;
|
|
349
|
-
}>;
|
|
350
|
-
}>;
|
|
351
|
-
};
|
|
286
|
+
keyword: string;
|
|
287
|
+
startTime?: Date;
|
|
288
|
+
endTime?: Date;
|
|
289
|
+
geo?: string;
|
|
290
|
+
resolution?: 'COUNTRY' | 'REGION' | 'CITY' | 'DMA';
|
|
291
|
+
hl?: string;
|
|
292
|
+
timezone?: number;
|
|
293
|
+
category?: number;
|
|
294
|
+
enableBackoff?: boolean;
|
|
352
295
|
}
|
|
353
296
|
```
|
|
354
297
|
|
|
355
|
-
###
|
|
298
|
+
### RelatedTopicsOptions
|
|
356
299
|
|
|
357
300
|
```typescript
|
|
358
|
-
interface
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
}>;
|
|
368
|
-
}>;
|
|
369
|
-
};
|
|
301
|
+
interface RelatedTopicsOptions {
|
|
302
|
+
keyword: string;
|
|
303
|
+
geo?: string;
|
|
304
|
+
startTime?: Date;
|
|
305
|
+
endTime?: Date;
|
|
306
|
+
category?: number;
|
|
307
|
+
property?: string;
|
|
308
|
+
hl?: string;
|
|
309
|
+
enableBackoff?: boolean;
|
|
370
310
|
}
|
|
371
311
|
```
|
|
372
312
|
|
|
373
|
-
###
|
|
313
|
+
### RelatedQueriesOptions
|
|
374
314
|
|
|
375
315
|
```typescript
|
|
376
|
-
interface
|
|
377
|
-
|
|
378
|
-
|
|
316
|
+
interface RelatedQueriesOptions {
|
|
317
|
+
keyword: string;
|
|
318
|
+
geo?: string;
|
|
319
|
+
startTime?: Date;
|
|
320
|
+
endTime?: Date;
|
|
321
|
+
category?: number;
|
|
322
|
+
property?: string;
|
|
323
|
+
hl?: string;
|
|
324
|
+
enableBackoff?: boolean;
|
|
379
325
|
}
|
|
380
326
|
```
|
|
381
327
|
|
|
@@ -383,6 +329,16 @@ interface RelatedData {
|
|
|
383
329
|
|
|
384
330
|
### Building
|
|
385
331
|
|
|
332
|
+
```bash
|
|
333
|
+
npm run build
|
|
386
334
|
```
|
|
387
335
|
|
|
336
|
+
### Testing
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
npm test
|
|
388
340
|
```
|
|
341
|
+
|
|
342
|
+
## License
|
|
343
|
+
|
|
344
|
+
MIT
|
package/dist/cjs/constants.js
CHANGED
|
@@ -37,17 +37,17 @@ exports.GOOGLE_TRENDS_MAPPER = {
|
|
|
37
37
|
headers: {},
|
|
38
38
|
},
|
|
39
39
|
[enums_js_1.GoogleTrendsEndpoints.relatedTopics]: {
|
|
40
|
-
path: '/trends/api/widgetdata/
|
|
40
|
+
path: '/trends/api/widgetdata/relatedsearches',
|
|
41
41
|
method: 'GET',
|
|
42
42
|
host: GOOGLE_TRENDS_BASE_URL,
|
|
43
|
-
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/
|
|
43
|
+
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/relatedsearches`,
|
|
44
44
|
headers: {},
|
|
45
45
|
},
|
|
46
46
|
[enums_js_1.GoogleTrendsEndpoints.relatedQueries]: {
|
|
47
|
-
path: '/trends/api/widgetdata/
|
|
47
|
+
path: '/trends/api/widgetdata/relatedsearches',
|
|
48
48
|
method: 'GET',
|
|
49
49
|
host: GOOGLE_TRENDS_BASE_URL,
|
|
50
|
-
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/
|
|
50
|
+
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/relatedsearches`,
|
|
51
51
|
headers: {},
|
|
52
52
|
},
|
|
53
53
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DailyTrendingTopics, DailyTrendingTopicsOptions, RealTimeTrendsOptions, ExploreOptions, ExploreResponse, InterestByRegionOptions, InterestByRegionResponse, GoogleTrendsResponse, GoogleTrendsError, RelatedTopicsResponse, RelatedQueriesResponse, RelatedData } from '../types/index.js';
|
|
1
|
+
import { DailyTrendingTopics, DailyTrendingTopicsOptions, RealTimeTrendsOptions, ExploreOptions, ExploreResponse, InterestByRegionOptions, InterestByRegionResponse, GoogleTrendsResponse, GoogleTrendsError, RelatedTopicsResponse, RelatedTopicsOptions, RelatedQueriesResponse, RelatedData, RelatedQueriesOptions } from '../types/index.js';
|
|
2
2
|
export declare class GoogleTrendsApi {
|
|
3
3
|
/**
|
|
4
4
|
* Get autocomplete suggestions for a keyword
|
|
@@ -23,8 +23,8 @@ export declare class GoogleTrendsApi {
|
|
|
23
23
|
error: GoogleTrendsError;
|
|
24
24
|
}>;
|
|
25
25
|
interestByRegion({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: InterestByRegionOptions): Promise<GoogleTrendsResponse<InterestByRegionResponse>>;
|
|
26
|
-
relatedTopics({ keyword, geo,
|
|
27
|
-
relatedQueries({ keyword, geo,
|
|
26
|
+
relatedTopics({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: RelatedTopicsOptions): Promise<GoogleTrendsResponse<RelatedTopicsResponse>>;
|
|
27
|
+
relatedQueries({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: RelatedQueriesOptions): Promise<GoogleTrendsResponse<RelatedQueriesResponse>>;
|
|
28
28
|
relatedData({ keyword, geo, time, category, property, hl, }: ExploreOptions): Promise<GoogleTrendsResponse<RelatedData>>;
|
|
29
29
|
}
|
|
30
30
|
declare const _default: GoogleTrendsApi;
|
|
@@ -207,19 +207,21 @@ class GoogleTrendsApi {
|
|
|
207
207
|
return { error: new GoogleTrendsError_js_1.UnknownError('Interest by region request failed') };
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
|
-
async relatedTopics({ keyword, geo = 'US',
|
|
210
|
+
async relatedTopics({ keyword, geo = 'US', startTime = new Date('2004-01-01'), endTime = new Date(), category = 0, property = '', hl = 'en-US', enableBackoff = false, }) {
|
|
211
211
|
try {
|
|
212
212
|
// Validate keyword
|
|
213
213
|
if (!keyword || keyword.trim() === '') {
|
|
214
214
|
return { error: new GoogleTrendsError_js_1.InvalidRequestError('Keyword is required') };
|
|
215
215
|
}
|
|
216
|
+
const keywordValue = Array.isArray(keyword) ? keyword[0] : keyword;
|
|
217
|
+
const geoValue = Array.isArray(geo) ? geo[0] : geo;
|
|
216
218
|
// Step 1: Call explore to get widget data and token
|
|
219
|
+
const timeValue = `${(0, format_js_1.formatDate)(startTime)} ${(0, format_js_1.formatDate)(endTime)}`;
|
|
217
220
|
const exploreResponse = await this.explore({
|
|
218
|
-
keyword,
|
|
219
|
-
geo,
|
|
220
|
-
time,
|
|
221
|
+
keyword: keywordValue,
|
|
222
|
+
geo: geoValue,
|
|
223
|
+
time: timeValue,
|
|
221
224
|
category,
|
|
222
|
-
property,
|
|
223
225
|
hl,
|
|
224
226
|
enableBackoff
|
|
225
227
|
});
|
|
@@ -235,40 +237,16 @@ class GoogleTrendsApi {
|
|
|
235
237
|
if (!relatedTopicsWidget) {
|
|
236
238
|
return { error: new GoogleTrendsError_js_1.ParseError('No related topics widget found in explore response') };
|
|
237
239
|
}
|
|
240
|
+
const requestFromWidget = {
|
|
241
|
+
...relatedTopicsWidget.request
|
|
242
|
+
};
|
|
238
243
|
// Step 3: Call the related topics API with or without token
|
|
239
244
|
const options = {
|
|
240
245
|
...constants_js_1.GOOGLE_TRENDS_MAPPER[enums_js_1.GoogleTrendsEndpoints.relatedTopics],
|
|
241
246
|
qs: {
|
|
242
247
|
hl,
|
|
243
248
|
tz: '240',
|
|
244
|
-
req: JSON.stringify(
|
|
245
|
-
restriction: {
|
|
246
|
-
geo: { country: geo },
|
|
247
|
-
time: time,
|
|
248
|
-
originalTimeRangeForExploreUrl: time,
|
|
249
|
-
complexKeywordsRestriction: {
|
|
250
|
-
keyword: [{
|
|
251
|
-
type: 'BROAD',
|
|
252
|
-
value: keyword
|
|
253
|
-
}]
|
|
254
|
-
}
|
|
255
|
-
},
|
|
256
|
-
keywordType: 'ENTITY',
|
|
257
|
-
metric: ['TOP', 'RISING'],
|
|
258
|
-
trendinessSettings: {
|
|
259
|
-
compareTime: time
|
|
260
|
-
},
|
|
261
|
-
requestOptions: {
|
|
262
|
-
property: property,
|
|
263
|
-
backend: 'CM',
|
|
264
|
-
category: category
|
|
265
|
-
},
|
|
266
|
-
language: hl.split('-')[0],
|
|
267
|
-
userCountryCode: geo,
|
|
268
|
-
userConfig: {
|
|
269
|
-
userType: 'USER_TYPE_LEGIT_USER'
|
|
270
|
-
}
|
|
271
|
-
}),
|
|
249
|
+
req: JSON.stringify(requestFromWidget),
|
|
272
250
|
...(relatedTopicsWidget.token && { token: relatedTopicsWidget.token })
|
|
273
251
|
}
|
|
274
252
|
};
|
|
@@ -279,11 +257,7 @@ class GoogleTrendsApi {
|
|
|
279
257
|
const data = JSON.parse(text.slice(5));
|
|
280
258
|
// Return the data in the expected format
|
|
281
259
|
return {
|
|
282
|
-
data:
|
|
283
|
-
default: {
|
|
284
|
-
rankedList: data.default?.rankedList || []
|
|
285
|
-
}
|
|
286
|
-
}
|
|
260
|
+
data: data.default?.rankedList || []
|
|
287
261
|
};
|
|
288
262
|
}
|
|
289
263
|
catch (parseError) {
|
|
@@ -300,32 +274,62 @@ class GoogleTrendsApi {
|
|
|
300
274
|
return { error: new GoogleTrendsError_js_1.UnknownError() };
|
|
301
275
|
}
|
|
302
276
|
}
|
|
303
|
-
async relatedQueries({ keyword, geo = 'US',
|
|
277
|
+
async relatedQueries({ keyword, geo = 'US', startTime = new Date('2004-01-01'), endTime = new Date(), category = 0, property = '', hl = 'en-US', enableBackoff = false, }) {
|
|
304
278
|
try {
|
|
305
279
|
// Validate keyword
|
|
306
280
|
if (!keyword || keyword.trim() === '') {
|
|
307
|
-
return { error: new GoogleTrendsError_js_1.
|
|
281
|
+
return { error: new GoogleTrendsError_js_1.InvalidRequestError('Keyword is required') };
|
|
308
282
|
}
|
|
309
|
-
const
|
|
310
|
-
|
|
311
|
-
|
|
283
|
+
const keywordValue = Array.isArray(keyword) ? keyword[0] : keyword;
|
|
284
|
+
const geoValue = Array.isArray(geo) ? geo[0] : geo;
|
|
285
|
+
// Step 1: Call explore to get widget data and token
|
|
286
|
+
const timeValue = `${(0, format_js_1.formatDate)(startTime)} ${(0, format_js_1.formatDate)(endTime)}`;
|
|
287
|
+
const exploreResponse = await this.explore({
|
|
288
|
+
keyword: keywordValue,
|
|
289
|
+
geo: geoValue,
|
|
290
|
+
time: timeValue,
|
|
291
|
+
category,
|
|
292
|
+
hl,
|
|
293
|
+
enableBackoff
|
|
294
|
+
});
|
|
295
|
+
if ('error' in exploreResponse) {
|
|
296
|
+
return { error: exploreResponse.error };
|
|
312
297
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
298
|
+
if (!exploreResponse.widgets || exploreResponse.widgets.length === 0) {
|
|
299
|
+
return { error: new GoogleTrendsError_js_1.ParseError('No widgets found in explore response. This might be due to Google blocking the request, invalid parameters, or network issues.') };
|
|
300
|
+
}
|
|
301
|
+
const relatedQueriesWidget = exploreResponse.widgets.find(widget => widget.id === 'RELATED_QUERIES') || null; // Fallback to first widget if no specific one found
|
|
302
|
+
if (!relatedQueriesWidget) {
|
|
303
|
+
return { error: new GoogleTrendsError_js_1.ParseError('No related queries widget found in explore response') };
|
|
304
|
+
}
|
|
305
|
+
const requestFromWidget = {
|
|
306
|
+
...relatedQueriesWidget.request
|
|
307
|
+
};
|
|
308
|
+
const options = {
|
|
309
|
+
...constants_js_1.GOOGLE_TRENDS_MAPPER[enums_js_1.GoogleTrendsEndpoints.relatedQueries],
|
|
310
|
+
qs: {
|
|
311
|
+
hl,
|
|
312
|
+
tz: '240',
|
|
313
|
+
req: JSON.stringify(requestFromWidget),
|
|
314
|
+
token: relatedQueriesWidget.token
|
|
327
315
|
}
|
|
328
316
|
};
|
|
317
|
+
const response = await (0, request_js_1.request)(options.url, options, enableBackoff);
|
|
318
|
+
const text = await response.text();
|
|
319
|
+
// Parse the response
|
|
320
|
+
try {
|
|
321
|
+
const data = JSON.parse(text.slice(5));
|
|
322
|
+
// Return the data in the expected format
|
|
323
|
+
return {
|
|
324
|
+
data: data.default?.rankedList || []
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
catch (parseError) {
|
|
328
|
+
if (parseError instanceof Error) {
|
|
329
|
+
return { error: new GoogleTrendsError_js_1.ParseError(`Failed to parse related queries response: ${parseError.message}`) };
|
|
330
|
+
}
|
|
331
|
+
return { error: new GoogleTrendsError_js_1.ParseError('Failed to parse related queries response') };
|
|
332
|
+
}
|
|
329
333
|
}
|
|
330
334
|
catch (error) {
|
|
331
335
|
if (error instanceof Error) {
|
|
@@ -17,14 +17,31 @@ async function runRequest(options, body, attempt) {
|
|
|
17
17
|
res.on('data', (data) => { chunk += data; });
|
|
18
18
|
res.on('end', async () => {
|
|
19
19
|
const hasSetCookie = !!res.headers['set-cookie']?.length;
|
|
20
|
+
if (hasSetCookie) {
|
|
21
|
+
// Only keep the name=value part of the cookie, discard attributes like Expires, Path, etc.
|
|
22
|
+
const newCookie = res.headers['set-cookie'][0].split(';')[0];
|
|
23
|
+
cookieVal = newCookie;
|
|
24
|
+
if (options.headers) {
|
|
25
|
+
options.headers['cookie'] = cookieVal;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
20
28
|
const isRateLimited = res.statusCode === 429 ||
|
|
21
29
|
chunk.includes('Error 429') ||
|
|
22
30
|
chunk.includes('Too Many Requests');
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
31
|
+
const isUnauthorized = res.statusCode === 401;
|
|
32
|
+
const isMovedPermanentlyOrRedirect = res.statusCode === 302;
|
|
33
|
+
if (isMovedPermanentlyOrRedirect) {
|
|
34
|
+
cookieVal = undefined;
|
|
35
|
+
if (options.headers) {
|
|
36
|
+
delete options.headers['cookie'];
|
|
27
37
|
}
|
|
38
|
+
if (attempt < MAX_RETRIES) {
|
|
39
|
+
const retryResponse = await runRequest(options, body, attempt + 1);
|
|
40
|
+
resolve(retryResponse);
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (isRateLimited || isUnauthorized) {
|
|
28
45
|
if (attempt < MAX_RETRIES) {
|
|
29
46
|
const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.floor(Math.random() * 250);
|
|
30
47
|
await sleep(delay);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export interface CookieJar {
|
|
2
|
+
NID?: string;
|
|
3
|
+
AEC?: string;
|
|
4
|
+
__Secure_BUCKET?: string;
|
|
5
|
+
OTZ?: string;
|
|
6
|
+
_ga?: string;
|
|
7
|
+
_gid?: string;
|
|
8
|
+
__utma?: string;
|
|
9
|
+
__utmz?: string;
|
|
10
|
+
[key: string]: string | undefined;
|
|
11
|
+
}
|
|
12
|
+
export interface SessionConfig {
|
|
13
|
+
autoRefresh?: boolean;
|
|
14
|
+
maxRetries?: number;
|
|
15
|
+
baseDelayMs?: number;
|
|
16
|
+
cookieRefreshInterval?: number;
|
|
17
|
+
initialCookies?: CookieJar | string;
|
|
18
|
+
}
|
|
19
|
+
export declare class SessionManager {
|
|
20
|
+
private cookies;
|
|
21
|
+
private lastRefresh;
|
|
22
|
+
private config;
|
|
23
|
+
private userAgents;
|
|
24
|
+
constructor(config?: SessionConfig);
|
|
25
|
+
initialize(): Promise<void>;
|
|
26
|
+
private parseCookieString;
|
|
27
|
+
private refreshSession;
|
|
28
|
+
private fetchCookies;
|
|
29
|
+
private parseCookies;
|
|
30
|
+
private serializeCookies;
|
|
31
|
+
private getRandomUserAgent;
|
|
32
|
+
getCookieHeader(): Promise<string>;
|
|
33
|
+
getRequestHeaders(): Record<string, string>;
|
|
34
|
+
updateFromSetCookie(setCookieHeaders: string[]): void;
|
|
35
|
+
getCookies(): CookieJar;
|
|
36
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SessionManager = void 0;
|
|
7
|
+
const https_1 = __importDefault(require("https"));
|
|
8
|
+
const url_1 = require("url");
|
|
9
|
+
class SessionManager {
|
|
10
|
+
constructor(config = {}) {
|
|
11
|
+
this.cookies = {};
|
|
12
|
+
this.lastRefresh = 0;
|
|
13
|
+
this.userAgents = [
|
|
14
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
15
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
16
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
17
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
18
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
|
|
19
|
+
];
|
|
20
|
+
this.config = {
|
|
21
|
+
autoRefresh: config.autoRefresh ?? true,
|
|
22
|
+
maxRetries: config.maxRetries ?? 3,
|
|
23
|
+
baseDelayMs: config.baseDelayMs ?? 750,
|
|
24
|
+
cookieRefreshInterval: config.cookieRefreshInterval ?? 30 * 60 * 1000 // 30 min
|
|
25
|
+
};
|
|
26
|
+
if (config.initialCookies) {
|
|
27
|
+
if (typeof config.initialCookies === 'string') {
|
|
28
|
+
this.cookies = this.parseCookieString(config.initialCookies);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
this.cookies = config.initialCookies;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async initialize() {
|
|
36
|
+
// If we already have essential cookies (like NID) provided manually,
|
|
37
|
+
// we can skip the initial fetch if desired, or verify them.
|
|
38
|
+
// For now, we'll only fetch if we don't have them or if autoRefresh implies we should.
|
|
39
|
+
if (Object.keys(this.cookies).length === 0) {
|
|
40
|
+
await this.refreshSession();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
parseCookieString(cookieStr) {
|
|
44
|
+
const cookies = {};
|
|
45
|
+
if (!cookieStr)
|
|
46
|
+
return cookies;
|
|
47
|
+
cookieStr.split(';').forEach(pair => {
|
|
48
|
+
const [key, value] = pair.split('=');
|
|
49
|
+
if (key && value) {
|
|
50
|
+
cookies[key.trim()] = value.trim();
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
return cookies;
|
|
54
|
+
}
|
|
55
|
+
async refreshSession() {
|
|
56
|
+
try {
|
|
57
|
+
// Step 1: Hit main trends page to get initial cookies
|
|
58
|
+
const mainCookies = await this.fetchCookies('https://trends.google.com/trends/');
|
|
59
|
+
// Step 2: Hit explore page to get additional auth cookies
|
|
60
|
+
const exploreCookies = await this.fetchCookies('https://trends.google.com/trends/explore?geo=US&q=test', mainCookies);
|
|
61
|
+
this.cookies = { ...this.cookies, ...mainCookies, ...exploreCookies };
|
|
62
|
+
this.lastRefresh = Date.now();
|
|
63
|
+
// console.log('[SessionManager] Session refreshed. Cookies:', Object.keys(this.cookies));
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
console.error('[SessionManager] Failed to refresh session:', error);
|
|
67
|
+
// Don't throw if we have fallback cookies, otherwise propagate
|
|
68
|
+
if (Object.keys(this.cookies).length === 0) {
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
fetchCookies(url, existingCookies) {
|
|
74
|
+
return new Promise((resolve, reject) => {
|
|
75
|
+
const parsedUrl = new url_1.URL(url);
|
|
76
|
+
const options = {
|
|
77
|
+
hostname: parsedUrl.hostname,
|
|
78
|
+
path: parsedUrl.pathname + parsedUrl.search,
|
|
79
|
+
method: 'GET',
|
|
80
|
+
headers: {
|
|
81
|
+
'User-Agent': this.getRandomUserAgent(),
|
|
82
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
83
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
84
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
85
|
+
'Connection': 'keep-alive',
|
|
86
|
+
'Upgrade-Insecure-Requests': '1',
|
|
87
|
+
'Sec-Fetch-Dest': 'document',
|
|
88
|
+
'Sec-Fetch-Mode': 'navigate',
|
|
89
|
+
'Sec-Fetch-Site': 'none',
|
|
90
|
+
...(existingCookies ? { Cookie: this.serializeCookies(existingCookies) } : {})
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
const req = https_1.default.request(options, (res) => {
|
|
94
|
+
const cookies = this.parseCookies(res.headers['set-cookie'] || []);
|
|
95
|
+
// Consume response body to free up memory/socket
|
|
96
|
+
res.on('data', () => { });
|
|
97
|
+
res.on('end', () => resolve(cookies));
|
|
98
|
+
});
|
|
99
|
+
req.on('error', reject);
|
|
100
|
+
req.setTimeout(10000, () => {
|
|
101
|
+
req.destroy();
|
|
102
|
+
reject(new Error('Request timeout'));
|
|
103
|
+
});
|
|
104
|
+
req.end();
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
parseCookies(setCookieHeaders) {
|
|
108
|
+
const cookies = {};
|
|
109
|
+
for (const header of setCookieHeaders) {
|
|
110
|
+
const [cookiePart] = header.split(';');
|
|
111
|
+
const [key, value] = cookiePart.split('=');
|
|
112
|
+
const normalizedKey = key.replace(/__Secure-/g, '__Secure_');
|
|
113
|
+
// We accept more than just specific keys to be robust, but we can filter if needed.
|
|
114
|
+
// Keeping the filter for now to avoid junk.
|
|
115
|
+
if (['NID', 'AEC', '__Secure_BUCKET', 'OTZ', '_ga', '_gid', '__utma', '__utmz'].includes(normalizedKey)) {
|
|
116
|
+
cookies[normalizedKey] = value;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return cookies;
|
|
120
|
+
}
|
|
121
|
+
serializeCookies(cookies) {
|
|
122
|
+
return Object.entries(cookies)
|
|
123
|
+
.filter(([_, v]) => v !== undefined)
|
|
124
|
+
.map(([k, v]) => {
|
|
125
|
+
// Convert back to __Secure- format for sending
|
|
126
|
+
const key = k.replace(/__Secure_/g, '__Secure-');
|
|
127
|
+
return `${key}=${v}`;
|
|
128
|
+
})
|
|
129
|
+
.join('; ');
|
|
130
|
+
}
|
|
131
|
+
getRandomUserAgent() {
|
|
132
|
+
return this.userAgents[Math.floor(Math.random() * this.userAgents.length)];
|
|
133
|
+
}
|
|
134
|
+
async getCookieHeader() {
|
|
135
|
+
// Auto-refresh if needed
|
|
136
|
+
if (this.config.autoRefresh) {
|
|
137
|
+
const timeSinceRefresh = Date.now() - this.lastRefresh;
|
|
138
|
+
if (this.lastRefresh > 0 && timeSinceRefresh > this.config.cookieRefreshInterval) {
|
|
139
|
+
// console.log('[SessionManager] Auto-refreshing session...');
|
|
140
|
+
await this.refreshSession(); // Background refresh? No, await to ensure valid session.
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return this.serializeCookies(this.cookies);
|
|
144
|
+
}
|
|
145
|
+
getRequestHeaders() {
|
|
146
|
+
return {
|
|
147
|
+
'User-Agent': this.getRandomUserAgent(),
|
|
148
|
+
'Accept': 'application/json, text/plain, */*',
|
|
149
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
150
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
151
|
+
'Referer': 'https://trends.google.com/',
|
|
152
|
+
'Origin': 'https://trends.google.com',
|
|
153
|
+
'Connection': 'keep-alive',
|
|
154
|
+
'Sec-Fetch-Dest': 'empty',
|
|
155
|
+
'Sec-Fetch-Mode': 'cors',
|
|
156
|
+
'Sec-Fetch-Site': 'same-origin',
|
|
157
|
+
'Cache-Control': 'no-cache',
|
|
158
|
+
'Pragma': 'no-cache'
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
// Manual cookie update from 429 responses
|
|
162
|
+
updateFromSetCookie(setCookieHeaders) {
|
|
163
|
+
const newCookies = this.parseCookies(setCookieHeaders);
|
|
164
|
+
this.cookies = { ...this.cookies, ...newCookies };
|
|
165
|
+
// console.log('[SessionManager] Updated cookies from response');
|
|
166
|
+
}
|
|
167
|
+
// For testing/debugging
|
|
168
|
+
getCookies() {
|
|
169
|
+
return { ...this.cookies };
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
exports.SessionManager = SessionManager;
|
package/dist/cjs/index.d.ts
CHANGED
|
@@ -6,8 +6,8 @@ export declare const explore: ({ keyword, geo, time, category, property, hl, ena
|
|
|
6
6
|
error: import("./types/index.js").GoogleTrendsError;
|
|
7
7
|
}>;
|
|
8
8
|
export declare const interestByRegion: ({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: import("./types/index.js").InterestByRegionOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").InterestByRegionResponse>>;
|
|
9
|
-
export declare const relatedTopics: ({ keyword, geo,
|
|
10
|
-
export declare const relatedQueries: ({ keyword, geo,
|
|
9
|
+
export declare const relatedTopics: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedTopicsOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedTopicsResponse>>;
|
|
10
|
+
export declare const relatedQueries: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedQueriesOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedQueriesResponse>>;
|
|
11
11
|
export declare const relatedData: ({ keyword, geo, time, category, property, hl, }: import("./types/index.js").ExploreOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedData>>;
|
|
12
12
|
export { GoogleTrendsApi };
|
|
13
13
|
declare const _default: {
|
|
@@ -18,8 +18,8 @@ declare const _default: {
|
|
|
18
18
|
error: import("./types/index.js").GoogleTrendsError;
|
|
19
19
|
}>;
|
|
20
20
|
interestByRegion: ({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: import("./types/index.js").InterestByRegionOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").InterestByRegionResponse>>;
|
|
21
|
-
relatedTopics: ({ keyword, geo,
|
|
22
|
-
relatedQueries: ({ keyword, geo,
|
|
21
|
+
relatedTopics: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedTopicsOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedTopicsResponse>>;
|
|
22
|
+
relatedQueries: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedQueriesOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedQueriesResponse>>;
|
|
23
23
|
relatedData: ({ keyword, geo, time, category, property, hl, }: import("./types/index.js").ExploreOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedData>>;
|
|
24
24
|
};
|
|
25
25
|
export default _default;
|
package/dist/esm/constants.js
CHANGED
|
@@ -34,17 +34,17 @@ export const GOOGLE_TRENDS_MAPPER = {
|
|
|
34
34
|
headers: {},
|
|
35
35
|
},
|
|
36
36
|
[GoogleTrendsEndpoints.relatedTopics]: {
|
|
37
|
-
path: '/trends/api/widgetdata/
|
|
37
|
+
path: '/trends/api/widgetdata/relatedsearches',
|
|
38
38
|
method: 'GET',
|
|
39
39
|
host: GOOGLE_TRENDS_BASE_URL,
|
|
40
|
-
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/
|
|
40
|
+
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/relatedsearches`,
|
|
41
41
|
headers: {},
|
|
42
42
|
},
|
|
43
43
|
[GoogleTrendsEndpoints.relatedQueries]: {
|
|
44
|
-
path: '/trends/api/widgetdata/
|
|
44
|
+
path: '/trends/api/widgetdata/relatedsearches',
|
|
45
45
|
method: 'GET',
|
|
46
46
|
host: GOOGLE_TRENDS_BASE_URL,
|
|
47
|
-
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/
|
|
47
|
+
url: `https://${GOOGLE_TRENDS_BASE_URL}/trends/api/widgetdata/relatedsearches`,
|
|
48
48
|
headers: {},
|
|
49
49
|
},
|
|
50
50
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DailyTrendingTopics, DailyTrendingTopicsOptions, RealTimeTrendsOptions, ExploreOptions, ExploreResponse, InterestByRegionOptions, InterestByRegionResponse, GoogleTrendsResponse, GoogleTrendsError, RelatedTopicsResponse, RelatedQueriesResponse, RelatedData } from '../types/index.js';
|
|
1
|
+
import { DailyTrendingTopics, DailyTrendingTopicsOptions, RealTimeTrendsOptions, ExploreOptions, ExploreResponse, InterestByRegionOptions, InterestByRegionResponse, GoogleTrendsResponse, GoogleTrendsError, RelatedTopicsResponse, RelatedTopicsOptions, RelatedQueriesResponse, RelatedData, RelatedQueriesOptions } from '../types/index.js';
|
|
2
2
|
export declare class GoogleTrendsApi {
|
|
3
3
|
/**
|
|
4
4
|
* Get autocomplete suggestions for a keyword
|
|
@@ -23,8 +23,8 @@ export declare class GoogleTrendsApi {
|
|
|
23
23
|
error: GoogleTrendsError;
|
|
24
24
|
}>;
|
|
25
25
|
interestByRegion({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: InterestByRegionOptions): Promise<GoogleTrendsResponse<InterestByRegionResponse>>;
|
|
26
|
-
relatedTopics({ keyword, geo,
|
|
27
|
-
relatedQueries({ keyword, geo,
|
|
26
|
+
relatedTopics({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: RelatedTopicsOptions): Promise<GoogleTrendsResponse<RelatedTopicsResponse>>;
|
|
27
|
+
relatedQueries({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: RelatedQueriesOptions): Promise<GoogleTrendsResponse<RelatedQueriesResponse>>;
|
|
28
28
|
relatedData({ keyword, geo, time, category, property, hl, }: ExploreOptions): Promise<GoogleTrendsResponse<RelatedData>>;
|
|
29
29
|
}
|
|
30
30
|
declare const _default: GoogleTrendsApi;
|
|
@@ -204,19 +204,21 @@ export class GoogleTrendsApi {
|
|
|
204
204
|
return { error: new UnknownError('Interest by region request failed') };
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
|
-
async relatedTopics({ keyword, geo = 'US',
|
|
207
|
+
async relatedTopics({ keyword, geo = 'US', startTime = new Date('2004-01-01'), endTime = new Date(), category = 0, property = '', hl = 'en-US', enableBackoff = false, }) {
|
|
208
208
|
try {
|
|
209
209
|
// Validate keyword
|
|
210
210
|
if (!keyword || keyword.trim() === '') {
|
|
211
211
|
return { error: new InvalidRequestError('Keyword is required') };
|
|
212
212
|
}
|
|
213
|
+
const keywordValue = Array.isArray(keyword) ? keyword[0] : keyword;
|
|
214
|
+
const geoValue = Array.isArray(geo) ? geo[0] : geo;
|
|
213
215
|
// Step 1: Call explore to get widget data and token
|
|
216
|
+
const timeValue = `${formatDate(startTime)} ${formatDate(endTime)}`;
|
|
214
217
|
const exploreResponse = await this.explore({
|
|
215
|
-
keyword,
|
|
216
|
-
geo,
|
|
217
|
-
time,
|
|
218
|
+
keyword: keywordValue,
|
|
219
|
+
geo: geoValue,
|
|
220
|
+
time: timeValue,
|
|
218
221
|
category,
|
|
219
|
-
property,
|
|
220
222
|
hl,
|
|
221
223
|
enableBackoff
|
|
222
224
|
});
|
|
@@ -232,40 +234,16 @@ export class GoogleTrendsApi {
|
|
|
232
234
|
if (!relatedTopicsWidget) {
|
|
233
235
|
return { error: new ParseError('No related topics widget found in explore response') };
|
|
234
236
|
}
|
|
237
|
+
const requestFromWidget = {
|
|
238
|
+
...relatedTopicsWidget.request
|
|
239
|
+
};
|
|
235
240
|
// Step 3: Call the related topics API with or without token
|
|
236
241
|
const options = {
|
|
237
242
|
...GOOGLE_TRENDS_MAPPER[GoogleTrendsEndpoints.relatedTopics],
|
|
238
243
|
qs: {
|
|
239
244
|
hl,
|
|
240
245
|
tz: '240',
|
|
241
|
-
req: JSON.stringify(
|
|
242
|
-
restriction: {
|
|
243
|
-
geo: { country: geo },
|
|
244
|
-
time: time,
|
|
245
|
-
originalTimeRangeForExploreUrl: time,
|
|
246
|
-
complexKeywordsRestriction: {
|
|
247
|
-
keyword: [{
|
|
248
|
-
type: 'BROAD',
|
|
249
|
-
value: keyword
|
|
250
|
-
}]
|
|
251
|
-
}
|
|
252
|
-
},
|
|
253
|
-
keywordType: 'ENTITY',
|
|
254
|
-
metric: ['TOP', 'RISING'],
|
|
255
|
-
trendinessSettings: {
|
|
256
|
-
compareTime: time
|
|
257
|
-
},
|
|
258
|
-
requestOptions: {
|
|
259
|
-
property: property,
|
|
260
|
-
backend: 'CM',
|
|
261
|
-
category: category
|
|
262
|
-
},
|
|
263
|
-
language: hl.split('-')[0],
|
|
264
|
-
userCountryCode: geo,
|
|
265
|
-
userConfig: {
|
|
266
|
-
userType: 'USER_TYPE_LEGIT_USER'
|
|
267
|
-
}
|
|
268
|
-
}),
|
|
246
|
+
req: JSON.stringify(requestFromWidget),
|
|
269
247
|
...(relatedTopicsWidget.token && { token: relatedTopicsWidget.token })
|
|
270
248
|
}
|
|
271
249
|
};
|
|
@@ -276,11 +254,7 @@ export class GoogleTrendsApi {
|
|
|
276
254
|
const data = JSON.parse(text.slice(5));
|
|
277
255
|
// Return the data in the expected format
|
|
278
256
|
return {
|
|
279
|
-
data:
|
|
280
|
-
default: {
|
|
281
|
-
rankedList: data.default?.rankedList || []
|
|
282
|
-
}
|
|
283
|
-
}
|
|
257
|
+
data: data.default?.rankedList || []
|
|
284
258
|
};
|
|
285
259
|
}
|
|
286
260
|
catch (parseError) {
|
|
@@ -297,32 +271,62 @@ export class GoogleTrendsApi {
|
|
|
297
271
|
return { error: new UnknownError() };
|
|
298
272
|
}
|
|
299
273
|
}
|
|
300
|
-
async relatedQueries({ keyword, geo = 'US',
|
|
274
|
+
async relatedQueries({ keyword, geo = 'US', startTime = new Date('2004-01-01'), endTime = new Date(), category = 0, property = '', hl = 'en-US', enableBackoff = false, }) {
|
|
301
275
|
try {
|
|
302
276
|
// Validate keyword
|
|
303
277
|
if (!keyword || keyword.trim() === '') {
|
|
304
|
-
return { error: new
|
|
278
|
+
return { error: new InvalidRequestError('Keyword is required') };
|
|
305
279
|
}
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
280
|
+
const keywordValue = Array.isArray(keyword) ? keyword[0] : keyword;
|
|
281
|
+
const geoValue = Array.isArray(geo) ? geo[0] : geo;
|
|
282
|
+
// Step 1: Call explore to get widget data and token
|
|
283
|
+
const timeValue = `${formatDate(startTime)} ${formatDate(endTime)}`;
|
|
284
|
+
const exploreResponse = await this.explore({
|
|
285
|
+
keyword: keywordValue,
|
|
286
|
+
geo: geoValue,
|
|
287
|
+
time: timeValue,
|
|
288
|
+
category,
|
|
289
|
+
hl,
|
|
290
|
+
enableBackoff
|
|
291
|
+
});
|
|
292
|
+
if ('error' in exploreResponse) {
|
|
293
|
+
return { error: exploreResponse.error };
|
|
309
294
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
295
|
+
if (!exploreResponse.widgets || exploreResponse.widgets.length === 0) {
|
|
296
|
+
return { error: new ParseError('No widgets found in explore response. This might be due to Google blocking the request, invalid parameters, or network issues.') };
|
|
297
|
+
}
|
|
298
|
+
const relatedQueriesWidget = exploreResponse.widgets.find(widget => widget.id === 'RELATED_QUERIES') || null; // Fallback to first widget if no specific one found
|
|
299
|
+
if (!relatedQueriesWidget) {
|
|
300
|
+
return { error: new ParseError('No related queries widget found in explore response') };
|
|
301
|
+
}
|
|
302
|
+
const requestFromWidget = {
|
|
303
|
+
...relatedQueriesWidget.request
|
|
304
|
+
};
|
|
305
|
+
const options = {
|
|
306
|
+
...GOOGLE_TRENDS_MAPPER[GoogleTrendsEndpoints.relatedQueries],
|
|
307
|
+
qs: {
|
|
308
|
+
hl,
|
|
309
|
+
tz: '240',
|
|
310
|
+
req: JSON.stringify(requestFromWidget),
|
|
311
|
+
token: relatedQueriesWidget.token
|
|
324
312
|
}
|
|
325
313
|
};
|
|
314
|
+
const response = await request(options.url, options, enableBackoff);
|
|
315
|
+
const text = await response.text();
|
|
316
|
+
// Parse the response
|
|
317
|
+
try {
|
|
318
|
+
const data = JSON.parse(text.slice(5));
|
|
319
|
+
// Return the data in the expected format
|
|
320
|
+
return {
|
|
321
|
+
data: data.default?.rankedList || []
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
catch (parseError) {
|
|
325
|
+
if (parseError instanceof Error) {
|
|
326
|
+
return { error: new ParseError(`Failed to parse related queries response: ${parseError.message}`) };
|
|
327
|
+
}
|
|
328
|
+
return { error: new ParseError('Failed to parse related queries response') };
|
|
329
|
+
}
|
|
326
330
|
}
|
|
327
331
|
catch (error) {
|
|
328
332
|
if (error instanceof Error) {
|
|
@@ -11,14 +11,31 @@ async function runRequest(options, body, attempt) {
|
|
|
11
11
|
res.on('data', (data) => { chunk += data; });
|
|
12
12
|
res.on('end', async () => {
|
|
13
13
|
const hasSetCookie = !!res.headers['set-cookie']?.length;
|
|
14
|
+
if (hasSetCookie) {
|
|
15
|
+
// Only keep the name=value part of the cookie, discard attributes like Expires, Path, etc.
|
|
16
|
+
const newCookie = res.headers['set-cookie'][0].split(';')[0];
|
|
17
|
+
cookieVal = newCookie;
|
|
18
|
+
if (options.headers) {
|
|
19
|
+
options.headers['cookie'] = cookieVal;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
14
22
|
const isRateLimited = res.statusCode === 429 ||
|
|
15
23
|
chunk.includes('Error 429') ||
|
|
16
24
|
chunk.includes('Too Many Requests');
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
25
|
+
const isUnauthorized = res.statusCode === 401;
|
|
26
|
+
const isMovedPermanentlyOrRedirect = res.statusCode === 302;
|
|
27
|
+
if (isMovedPermanentlyOrRedirect) {
|
|
28
|
+
cookieVal = undefined;
|
|
29
|
+
if (options.headers) {
|
|
30
|
+
delete options.headers['cookie'];
|
|
21
31
|
}
|
|
32
|
+
if (attempt < MAX_RETRIES) {
|
|
33
|
+
const retryResponse = await runRequest(options, body, attempt + 1);
|
|
34
|
+
resolve(retryResponse);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (isRateLimited || isUnauthorized) {
|
|
22
39
|
if (attempt < MAX_RETRIES) {
|
|
23
40
|
const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.floor(Math.random() * 250);
|
|
24
41
|
await sleep(delay);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export interface CookieJar {
|
|
2
|
+
NID?: string;
|
|
3
|
+
AEC?: string;
|
|
4
|
+
__Secure_BUCKET?: string;
|
|
5
|
+
OTZ?: string;
|
|
6
|
+
_ga?: string;
|
|
7
|
+
_gid?: string;
|
|
8
|
+
__utma?: string;
|
|
9
|
+
__utmz?: string;
|
|
10
|
+
[key: string]: string | undefined;
|
|
11
|
+
}
|
|
12
|
+
export interface SessionConfig {
|
|
13
|
+
autoRefresh?: boolean;
|
|
14
|
+
maxRetries?: number;
|
|
15
|
+
baseDelayMs?: number;
|
|
16
|
+
cookieRefreshInterval?: number;
|
|
17
|
+
initialCookies?: CookieJar | string;
|
|
18
|
+
}
|
|
19
|
+
export declare class SessionManager {
|
|
20
|
+
private cookies;
|
|
21
|
+
private lastRefresh;
|
|
22
|
+
private config;
|
|
23
|
+
private userAgents;
|
|
24
|
+
constructor(config?: SessionConfig);
|
|
25
|
+
initialize(): Promise<void>;
|
|
26
|
+
private parseCookieString;
|
|
27
|
+
private refreshSession;
|
|
28
|
+
private fetchCookies;
|
|
29
|
+
private parseCookies;
|
|
30
|
+
private serializeCookies;
|
|
31
|
+
private getRandomUserAgent;
|
|
32
|
+
getCookieHeader(): Promise<string>;
|
|
33
|
+
getRequestHeaders(): Record<string, string>;
|
|
34
|
+
updateFromSetCookie(setCookieHeaders: string[]): void;
|
|
35
|
+
getCookies(): CookieJar;
|
|
36
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import https from 'https';
|
|
2
|
+
import { URL } from 'url';
|
|
3
|
+
export class SessionManager {
|
|
4
|
+
constructor(config = {}) {
|
|
5
|
+
this.cookies = {};
|
|
6
|
+
this.lastRefresh = 0;
|
|
7
|
+
this.userAgents = [
|
|
8
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
9
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
10
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
11
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
12
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15'
|
|
13
|
+
];
|
|
14
|
+
this.config = {
|
|
15
|
+
autoRefresh: config.autoRefresh ?? true,
|
|
16
|
+
maxRetries: config.maxRetries ?? 3,
|
|
17
|
+
baseDelayMs: config.baseDelayMs ?? 750,
|
|
18
|
+
cookieRefreshInterval: config.cookieRefreshInterval ?? 30 * 60 * 1000 // 30 min
|
|
19
|
+
};
|
|
20
|
+
if (config.initialCookies) {
|
|
21
|
+
if (typeof config.initialCookies === 'string') {
|
|
22
|
+
this.cookies = this.parseCookieString(config.initialCookies);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
this.cookies = config.initialCookies;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
async initialize() {
|
|
30
|
+
// If we already have essential cookies (like NID) provided manually,
|
|
31
|
+
// we can skip the initial fetch if desired, or verify them.
|
|
32
|
+
// For now, we'll only fetch if we don't have them or if autoRefresh implies we should.
|
|
33
|
+
if (Object.keys(this.cookies).length === 0) {
|
|
34
|
+
await this.refreshSession();
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
parseCookieString(cookieStr) {
|
|
38
|
+
const cookies = {};
|
|
39
|
+
if (!cookieStr)
|
|
40
|
+
return cookies;
|
|
41
|
+
cookieStr.split(';').forEach(pair => {
|
|
42
|
+
const [key, value] = pair.split('=');
|
|
43
|
+
if (key && value) {
|
|
44
|
+
cookies[key.trim()] = value.trim();
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
return cookies;
|
|
48
|
+
}
|
|
49
|
+
async refreshSession() {
|
|
50
|
+
try {
|
|
51
|
+
// Step 1: Hit main trends page to get initial cookies
|
|
52
|
+
const mainCookies = await this.fetchCookies('https://trends.google.com/trends/');
|
|
53
|
+
// Step 2: Hit explore page to get additional auth cookies
|
|
54
|
+
const exploreCookies = await this.fetchCookies('https://trends.google.com/trends/explore?geo=US&q=test', mainCookies);
|
|
55
|
+
this.cookies = { ...this.cookies, ...mainCookies, ...exploreCookies };
|
|
56
|
+
this.lastRefresh = Date.now();
|
|
57
|
+
// console.log('[SessionManager] Session refreshed. Cookies:', Object.keys(this.cookies));
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
console.error('[SessionManager] Failed to refresh session:', error);
|
|
61
|
+
// Don't throw if we have fallback cookies, otherwise propagate
|
|
62
|
+
if (Object.keys(this.cookies).length === 0) {
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
fetchCookies(url, existingCookies) {
|
|
68
|
+
return new Promise((resolve, reject) => {
|
|
69
|
+
const parsedUrl = new URL(url);
|
|
70
|
+
const options = {
|
|
71
|
+
hostname: parsedUrl.hostname,
|
|
72
|
+
path: parsedUrl.pathname + parsedUrl.search,
|
|
73
|
+
method: 'GET',
|
|
74
|
+
headers: {
|
|
75
|
+
'User-Agent': this.getRandomUserAgent(),
|
|
76
|
+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
77
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
78
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
79
|
+
'Connection': 'keep-alive',
|
|
80
|
+
'Upgrade-Insecure-Requests': '1',
|
|
81
|
+
'Sec-Fetch-Dest': 'document',
|
|
82
|
+
'Sec-Fetch-Mode': 'navigate',
|
|
83
|
+
'Sec-Fetch-Site': 'none',
|
|
84
|
+
...(existingCookies ? { Cookie: this.serializeCookies(existingCookies) } : {})
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
const req = https.request(options, (res) => {
|
|
88
|
+
const cookies = this.parseCookies(res.headers['set-cookie'] || []);
|
|
89
|
+
// Consume response body to free up memory/socket
|
|
90
|
+
res.on('data', () => { });
|
|
91
|
+
res.on('end', () => resolve(cookies));
|
|
92
|
+
});
|
|
93
|
+
req.on('error', reject);
|
|
94
|
+
req.setTimeout(10000, () => {
|
|
95
|
+
req.destroy();
|
|
96
|
+
reject(new Error('Request timeout'));
|
|
97
|
+
});
|
|
98
|
+
req.end();
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
parseCookies(setCookieHeaders) {
|
|
102
|
+
const cookies = {};
|
|
103
|
+
for (const header of setCookieHeaders) {
|
|
104
|
+
const [cookiePart] = header.split(';');
|
|
105
|
+
const [key, value] = cookiePart.split('=');
|
|
106
|
+
const normalizedKey = key.replace(/__Secure-/g, '__Secure_');
|
|
107
|
+
// We accept more than just specific keys to be robust, but we can filter if needed.
|
|
108
|
+
// Keeping the filter for now to avoid junk.
|
|
109
|
+
if (['NID', 'AEC', '__Secure_BUCKET', 'OTZ', '_ga', '_gid', '__utma', '__utmz'].includes(normalizedKey)) {
|
|
110
|
+
cookies[normalizedKey] = value;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return cookies;
|
|
114
|
+
}
|
|
115
|
+
serializeCookies(cookies) {
|
|
116
|
+
return Object.entries(cookies)
|
|
117
|
+
.filter(([_, v]) => v !== undefined)
|
|
118
|
+
.map(([k, v]) => {
|
|
119
|
+
// Convert back to __Secure- format for sending
|
|
120
|
+
const key = k.replace(/__Secure_/g, '__Secure-');
|
|
121
|
+
return `${key}=${v}`;
|
|
122
|
+
})
|
|
123
|
+
.join('; ');
|
|
124
|
+
}
|
|
125
|
+
getRandomUserAgent() {
|
|
126
|
+
return this.userAgents[Math.floor(Math.random() * this.userAgents.length)];
|
|
127
|
+
}
|
|
128
|
+
async getCookieHeader() {
|
|
129
|
+
// Auto-refresh if needed
|
|
130
|
+
if (this.config.autoRefresh) {
|
|
131
|
+
const timeSinceRefresh = Date.now() - this.lastRefresh;
|
|
132
|
+
if (this.lastRefresh > 0 && timeSinceRefresh > this.config.cookieRefreshInterval) {
|
|
133
|
+
// console.log('[SessionManager] Auto-refreshing session...');
|
|
134
|
+
await this.refreshSession(); // Background refresh? No, await to ensure valid session.
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return this.serializeCookies(this.cookies);
|
|
138
|
+
}
|
|
139
|
+
getRequestHeaders() {
|
|
140
|
+
return {
|
|
141
|
+
'User-Agent': this.getRandomUserAgent(),
|
|
142
|
+
'Accept': 'application/json, text/plain, */*',
|
|
143
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
144
|
+
'Accept-Encoding': 'gzip, deflate, br',
|
|
145
|
+
'Referer': 'https://trends.google.com/',
|
|
146
|
+
'Origin': 'https://trends.google.com',
|
|
147
|
+
'Connection': 'keep-alive',
|
|
148
|
+
'Sec-Fetch-Dest': 'empty',
|
|
149
|
+
'Sec-Fetch-Mode': 'cors',
|
|
150
|
+
'Sec-Fetch-Site': 'same-origin',
|
|
151
|
+
'Cache-Control': 'no-cache',
|
|
152
|
+
'Pragma': 'no-cache'
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
// Manual cookie update from 429 responses
|
|
156
|
+
updateFromSetCookie(setCookieHeaders) {
|
|
157
|
+
const newCookies = this.parseCookies(setCookieHeaders);
|
|
158
|
+
this.cookies = { ...this.cookies, ...newCookies };
|
|
159
|
+
// console.log('[SessionManager] Updated cookies from response');
|
|
160
|
+
}
|
|
161
|
+
// For testing/debugging
|
|
162
|
+
getCookies() {
|
|
163
|
+
return { ...this.cookies };
|
|
164
|
+
}
|
|
165
|
+
}
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -6,8 +6,8 @@ export declare const explore: ({ keyword, geo, time, category, property, hl, ena
|
|
|
6
6
|
error: import("./types/index.js").GoogleTrendsError;
|
|
7
7
|
}>;
|
|
8
8
|
export declare const interestByRegion: ({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: import("./types/index.js").InterestByRegionOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").InterestByRegionResponse>>;
|
|
9
|
-
export declare const relatedTopics: ({ keyword, geo,
|
|
10
|
-
export declare const relatedQueries: ({ keyword, geo,
|
|
9
|
+
export declare const relatedTopics: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedTopicsOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedTopicsResponse>>;
|
|
10
|
+
export declare const relatedQueries: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedQueriesOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedQueriesResponse>>;
|
|
11
11
|
export declare const relatedData: ({ keyword, geo, time, category, property, hl, }: import("./types/index.js").ExploreOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedData>>;
|
|
12
12
|
export { GoogleTrendsApi };
|
|
13
13
|
declare const _default: {
|
|
@@ -18,8 +18,8 @@ declare const _default: {
|
|
|
18
18
|
error: import("./types/index.js").GoogleTrendsError;
|
|
19
19
|
}>;
|
|
20
20
|
interestByRegion: ({ keyword, startTime, endTime, geo, resolution, hl, timezone, category, enableBackoff }: import("./types/index.js").InterestByRegionOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").InterestByRegionResponse>>;
|
|
21
|
-
relatedTopics: ({ keyword, geo,
|
|
22
|
-
relatedQueries: ({ keyword, geo,
|
|
21
|
+
relatedTopics: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedTopicsOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedTopicsResponse>>;
|
|
22
|
+
relatedQueries: ({ keyword, geo, startTime, endTime, category, property, hl, enableBackoff, }: import("./types/index.js").RelatedQueriesOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedQueriesResponse>>;
|
|
23
23
|
relatedData: ({ keyword, geo, time, category, property, hl, }: import("./types/index.js").ExploreOptions) => Promise<import("./types/index.js").GoogleTrendsResponse<import("./types/index.js").RelatedData>>;
|
|
24
24
|
};
|
|
25
25
|
export default _default;
|