@the-convocation/twitter-scraper 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +8 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +22 -1
- package/dist/api.js.map +1 -1
- package/dist/scraper.d.ts +16 -15
- package/dist/scraper.d.ts.map +1 -1
- package/dist/scraper.js +16 -15
- package/dist/scraper.js.map +1 -1
- package/package.json +2 -2
- package/typedoc.json +6 -0
package/dist/api.d.ts
CHANGED
|
@@ -11,6 +11,14 @@ export type RequestApiResult<T> = {
|
|
|
11
11
|
success: false;
|
|
12
12
|
err: Error;
|
|
13
13
|
};
|
|
14
|
+
/**
|
|
15
|
+
* Used internally to send HTTP requests to the Twitter API.
|
|
16
|
+
* @internal
|
|
17
|
+
*
|
|
18
|
+
* @param url - The URL to send the request to.
|
|
19
|
+
* @param auth - The instance of {@link TwitterAuth} that will be used to authorize this request.
|
|
20
|
+
* @param method - The HTTP method used when sending this request.
|
|
21
|
+
*/
|
|
14
22
|
export declare function requestApi<T>(url: string, auth: TwitterAuth, method?: 'GET' | 'POST'): Promise<RequestApiResult<T>>;
|
|
15
23
|
export declare function addApiFeatures(o: object): {
|
|
16
24
|
rweb_lists_timeline_redesign_enabled: boolean;
|
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAMrC,eAAO,MAAM,WAAW,uHAC8F,CAAC;AACvH,eAAO,MAAM,YAAY,6GACmF,CAAC;AAE7G;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAC1B;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAC3B;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,KAAK,CAAA;CAAE,CAAC;AAEnC,wBAAsB,UAAU,CAAC,CAAC,EAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,MAAM,GAAE,KAAK,GAAG,MAAc,GAC7B,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AAMrC,eAAO,MAAM,WAAW,uHAC8F,CAAC;AACvH,eAAO,MAAM,YAAY,6GACmF,CAAC;AAE7G;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,CAAC,IAC1B;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAC3B;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,KAAK,CAAA;CAAE,CAAC;AAEnC;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAAC,CAAC,EAChC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,MAAM,GAAE,KAAK,GAAG,MAAc,GAC7B,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAyD9B;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;EAoBvC;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,eAAe,EACvB,mBAAmB,EAAE,OAAO,GAC3B,eAAe,CAoCjB"}
|
package/dist/api.js
CHANGED
|
@@ -10,6 +10,14 @@ const headers_polyfill_1 = require("headers-polyfill");
|
|
|
10
10
|
const cross_fetch_1 = __importDefault(require("cross-fetch"));
|
|
11
11
|
exports.bearerToken = 'AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw';
|
|
12
12
|
exports.bearerToken2 = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
13
|
+
/**
|
|
14
|
+
* Used internally to send HTTP requests to the Twitter API.
|
|
15
|
+
* @internal
|
|
16
|
+
*
|
|
17
|
+
* @param url - The URL to send the request to.
|
|
18
|
+
* @param auth - The instance of {@link TwitterAuth} that will be used to authorize this request.
|
|
19
|
+
* @param method - The HTTP method used when sending this request.
|
|
20
|
+
*/
|
|
13
21
|
async function requestApi(url, auth, method = 'GET') {
|
|
14
22
|
const headers = new headers_polyfill_1.Headers();
|
|
15
23
|
await auth.installTo(headers, url);
|
|
@@ -32,7 +40,20 @@ async function requestApi(url, auth, method = 'GET') {
|
|
|
32
40
|
}
|
|
33
41
|
await (0, requests_1.updateCookieJar)(auth.cookieJar(), res.headers);
|
|
34
42
|
if (res.status === 429) {
|
|
35
|
-
|
|
43
|
+
/*
|
|
44
|
+
Known headers at this point:
|
|
45
|
+
- x-rate-limit-limit: Maximum number of requests per time period?
|
|
46
|
+
- x-rate-limit-reset: UNIX timestamp when the current rate limit will be reset.
|
|
47
|
+
- x-rate-limit-remaining: Number of requests remaining in current time period?
|
|
48
|
+
*/
|
|
49
|
+
const xRateLimitRemaining = res.headers.get('x-rate-limit-remaining');
|
|
50
|
+
const xRateLimitReset = res.headers.get('x-rate-limit-reset');
|
|
51
|
+
if (xRateLimitRemaining == '0' && xRateLimitReset) {
|
|
52
|
+
const currentTime = new Date().valueOf() / 1000;
|
|
53
|
+
const timeDeltaMs = 1000 * (parseInt(xRateLimitReset) - currentTime);
|
|
54
|
+
// I have seen this block for 800s (~13 *minutes*)
|
|
55
|
+
await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
|
|
56
|
+
}
|
|
36
57
|
}
|
|
37
58
|
} while (res.status === 429);
|
|
38
59
|
if (!res.ok) {
|
package/dist/api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":";;;;;;AACA,qCAAoC;AACpC,yCAA6C;AAC7C,uDAA2C;AAC3C,8DAAgC;AAEnB,QAAA,WAAW,GACtB,oHAAoH,CAAC;AAC1G,QAAA,YAAY,GACvB,0GAA0G,CAAC;
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":";;;;;;AACA,qCAAoC;AACpC,yCAA6C;AAC7C,uDAA2C;AAC3C,8DAAgC;AAEnB,QAAA,WAAW,GACtB,oHAAoH,CAAC;AAC1G,QAAA,YAAY,GACvB,0GAA0G,CAAC;AAS7G;;;;;;;GAOG;AACI,KAAK,UAAU,UAAU,CAC9B,GAAW,EACX,IAAiB,EACjB,SAAyB,KAAK;IAE9B,MAAM,OAAO,GAAG,IAAI,0BAAO,EAAE,CAAC;IAC9B,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEnC,IAAI,GAAa,CAAC;IAClB,GAAG;QACD,IAAI;YACF,GAAG,GAAG,MAAM,IAAA,qBAAK,EAAC,GAAG,EAAE;gBACrB,MAAM;gBACN,OAAO;aACR,CAAC,CAAC;SACJ;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC,EAAE;gBAC3B,MAAM,GAAG,CAAC;aACX;YAED,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,GAAG,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC;aAC7C,CAAC;SACH;QAED,MAAM,IAAA,0BAAe,EAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAErD,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;YACtB;;;;;cAKE;YACF,MAAM,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAC9D,IAAI,mBAAmB,IAAI,GAAG,IAAI,eAAe,EAAE;gBACjD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;gBAChD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,WAAW,CAAC,CAAC;gBAErE,kDAAkD;gBAClD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;aAClE;SACF;KACF,QAAQ,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;IAE7B,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE;QACX,OAAO;YACL,OAAO,EAAE,KAAK;YACd,GAAG,EAAE,IAAI,iBAAQ,CAAC,GAAG,EAAE,oBAAoB,GAAG,CAAC,MAAM,EAAE,CAAC;SACzD,CAAC;KACH;IAED,MAAM,KAAK,GAAM,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,IAAI,GAAG,EAAE;QACnD,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACjC;SAAM;QACL,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;KACjC;AACH,CAAC;AA7DD,gCA6DC;AAED,SAAgB,cAAc,CAAC,CAAS;IACtC,OAAO;QACL,GAAG,CAAC;QACJ,oCAAoC,EAAE,IAAI;QAC1C,gDAAgD,EAAE,IAAI;QACtD,4BAA4B,EAAE,KAAK;QACnC,+CAA+C,EAAE,IAAI;QACrD,kDAAkD,EAAE,IAAI;QACxD,iEAAiE,EAAE,KAAK;QACxE,wCAAwC,EAAE,IAAI;QAC9C,qCAAqC,EAAE,IAAI;QAC3C,0DAA0D,EAAE,IAAI;QAChE,kCAAkC,EAAE,IAAI;QACxC,uCAAuC,EAAE,IAAI;QAC7C,gCAAgC,EAAE,KAAK;QACvC,yCAAyC,EAAE,IAAI;QAC/C,2BAA2B,EAAE,IAAI;QACjC,0CAA0C,EAAE,IAAI;QAChD,oCAAoC,EAAE,KAAK;KAC5C,CAAC;AACJ,CAAC;AApBD,wCAoBC;AAED,SAAgB,YAAY,CAC1B,MAAuB,EACvB,mBAA4B;IAE5B,MAAM,CAAC,GAAG,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,CAAC,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,CAAC,8BAA8B,EAAE,GAAG,CAAC,CAAC;IAChD,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;IAC7C,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IAC/B,MAAM,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;IAC3C,MAAM,CAAC,GAAG,CAAC,oCAAoC,EAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,CAAC,GAAG,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,GAAG,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,CAAC,GAAG,CAAC,qCAAqC,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,CAAC,GAAG,CAAC,sCAAsC,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,GAAG,CAAC,uBAAuB,EAAE,GAAG,mBAAmB,EAAE,CAAC,CAAC;IAC9D,MAAM,CAAC,GAAG,CACR,KAAK,EACL,iJAAiJ,CAClJ,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAvCD,oCAuCC"}
|
package/dist/scraper.d.ts
CHANGED
|
@@ -5,20 +5,21 @@ import { QueryProfilesResponse, QueryTweetsResponse } from './timeline-v1';
|
|
|
5
5
|
import { Tweet } from './tweets';
|
|
6
6
|
/**
|
|
7
7
|
* An interface to Twitter's undocumented API.
|
|
8
|
-
* Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
8
|
+
* - Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
9
9
|
*/
|
|
10
10
|
export declare class Scraper {
|
|
11
11
|
private auth;
|
|
12
12
|
private authTrends;
|
|
13
13
|
/**
|
|
14
|
-
* Creates a new Scraper object.
|
|
15
|
-
*
|
|
14
|
+
* Creates a new Scraper object.
|
|
15
|
+
* - Scrapers maintain their own guest tokens for Twitter's internal API.
|
|
16
|
+
* - Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
16
17
|
*/
|
|
17
18
|
constructor();
|
|
18
19
|
/**
|
|
19
20
|
* Fetches a Twitter profile.
|
|
20
21
|
* @param username The Twitter username of the profile to fetch, without an `@` at the beginning.
|
|
21
|
-
* @returns The requested
|
|
22
|
+
* @returns The requested {@link Profile}.
|
|
22
23
|
*/
|
|
23
24
|
getProfile(username: string): Promise<Profile>;
|
|
24
25
|
/**
|
|
@@ -33,14 +34,14 @@ export declare class Scraper {
|
|
|
33
34
|
* @param maxTweets The maximum number of tweets to return.
|
|
34
35
|
* @param includeReplies Whether or not replies should be included in the response.
|
|
35
36
|
* @param searchMode The category filter to apply to the search. Defaults to `Top`.
|
|
36
|
-
* @returns An
|
|
37
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filters.
|
|
37
38
|
*/
|
|
38
39
|
searchTweets(query: string, maxTweets: number, searchMode?: SearchMode): AsyncGenerator<Tweet, void>;
|
|
39
40
|
/**
|
|
40
41
|
* Fetches profiles from Twitter.
|
|
41
42
|
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
42
43
|
* @param maxProfiles The maximum number of profiles to return.
|
|
43
|
-
* @returns An
|
|
44
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filter(s).
|
|
44
45
|
*/
|
|
45
46
|
searchProfiles(query: string, maxProfiles: number): AsyncGenerator<Profile, void>;
|
|
46
47
|
/**
|
|
@@ -69,28 +70,28 @@ export declare class Scraper {
|
|
|
69
70
|
/**
|
|
70
71
|
* Fetches tweets from a Twitter user.
|
|
71
72
|
* @param user The user whose tweets should be returned.
|
|
72
|
-
* @param maxTweets The maximum number of tweets to return.
|
|
73
|
-
* @returns An
|
|
73
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
74
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
74
75
|
*/
|
|
75
|
-
getTweets(user: string, maxTweets
|
|
76
|
+
getTweets(user: string, maxTweets?: number): AsyncGenerator<Tweet>;
|
|
76
77
|
/**
|
|
77
78
|
* Fetches tweets from a Twitter user using their ID.
|
|
78
79
|
* @param userId The user whose tweets should be returned.
|
|
79
|
-
* @param maxTweets The maximum number of tweets to return.
|
|
80
|
-
* @returns An
|
|
80
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
81
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
81
82
|
*/
|
|
82
|
-
getTweetsByUserId(userId: string, maxTweets
|
|
83
|
+
getTweetsByUserId(userId: string, maxTweets?: number): AsyncGenerator<Tweet, void>;
|
|
83
84
|
/**
|
|
84
85
|
* Fetches the most recent tweet from a Twitter user.
|
|
85
86
|
* @param user The user whose latest tweet should be returned.
|
|
86
|
-
* @param includeRetweets Whether or not to include retweets.
|
|
87
|
+
* @param includeRetweets Whether or not to include retweets. Defaults to `false`.
|
|
87
88
|
* @returns The {@link Tweet} object or `null`/`undefined` if it couldn't be fetched.
|
|
88
89
|
*/
|
|
89
|
-
getLatestTweet(user: string, includeRetweets
|
|
90
|
+
getLatestTweet(user: string, includeRetweets?: boolean): Promise<Tweet | null | void>;
|
|
90
91
|
/**
|
|
91
92
|
* Fetches a single tweet.
|
|
92
93
|
* @param id The ID of the tweet to fetch.
|
|
93
|
-
* @returns The
|
|
94
|
+
* @returns The {@link Tweet} object, or `null` if it couldn't be fetched.
|
|
94
95
|
*/
|
|
95
96
|
getTweet(id: string): Promise<Tweet | null>;
|
|
96
97
|
/**
|
package/dist/scraper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAItC,OAAO,EAAqC,OAAO,EAAE,MAAM,WAAW,CAAC;AACvE,OAAO,EAGL,UAAU,EAGX,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAIL,KAAK,EAEN,MAAM,UAAU,CAAC;AAIlB;;;GAGG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,UAAU,CAAc;IAEhC
|
|
1
|
+
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAItC,OAAO,EAAqC,OAAO,EAAE,MAAM,WAAW,CAAC;AACvE,OAAO,EAGL,UAAU,EAGX,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAIL,KAAK,EAEN,MAAM,UAAU,CAAC;AAIlB;;;GAGG;AACH,qBAAa,OAAO;IAClB,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,UAAU,CAAc;IAEhC;;;;OAIG;;IAMH;;;;OAIG;IACU,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAK3D;;;;OAIG;IACU,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKvE;;;;;;;OAOG;IACI,YAAY,CACjB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,GAAE,UAA2B,GACtC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC;IAI9B;;;;;OAKG;IACI,cAAc,CACnB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GAClB,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;IAIhC;;;;;;;;OAQG;IACI,iBAAiB,CACtB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,UAAU,EACtB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,mBAAmB,CAAC;IAI/B;;;;;;OAMG;IACI,mBAAmB,CACxB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,qBAAqB,CAAC;IAIjC;;;OAGG;IACI,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAIrC;;;;;OAKG;IACI,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,cAAc,CAAC,KAAK,CAAC;IAItE;;;;;OAKG;IACI,iBAAiB,CACtB,MAAM,EAAE,MAAM,EACd,SAAS,SAAM,GACd,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC;IAI9B;;;;;OAKG;IACI,cAAc,CACnB,IAAI,EAAE,MAAM,EACZ,eAAe,UAAQ,GACtB,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC;IAI/B;;;;OAIG;IACI,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC;IAIlD;;;OAGG;IACI,aAAa,IAAI,OAAO;IAI/B;;;OAGG;IACU,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAM3C;;;;;;OAMG;IACU,KAAK,CAChB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IAQhB;;OAEG;IACU,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IASpC;;;OAGG;IACU,UAAU,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAI5C;;;OAGG;IACU,UAAU,CAAC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUpE;;OAEG;IACU,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAK1C;;;;;OAKG;IAEI,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAO3C;;;;;OAKG;IAEI,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAO9C,OAAO,CAAC,cAAc;CAOvB"}
|
package/dist/scraper.js
CHANGED
|
@@ -11,12 +11,13 @@ const tweets_1 = require("./tweets");
|
|
|
11
11
|
const twUrl = 'https://twitter.com';
|
|
12
12
|
/**
|
|
13
13
|
* An interface to Twitter's undocumented API.
|
|
14
|
-
* Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
14
|
+
* - Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
15
15
|
*/
|
|
16
16
|
class Scraper {
|
|
17
17
|
/**
|
|
18
|
-
* Creates a new Scraper object.
|
|
19
|
-
*
|
|
18
|
+
* Creates a new Scraper object.
|
|
19
|
+
* - Scrapers maintain their own guest tokens for Twitter's internal API.
|
|
20
|
+
* - Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
20
21
|
*/
|
|
21
22
|
constructor() {
|
|
22
23
|
this.auth = new auth_1.TwitterGuestAuth(api_1.bearerToken);
|
|
@@ -25,7 +26,7 @@ class Scraper {
|
|
|
25
26
|
/**
|
|
26
27
|
* Fetches a Twitter profile.
|
|
27
28
|
* @param username The Twitter username of the profile to fetch, without an `@` at the beginning.
|
|
28
|
-
* @returns The requested
|
|
29
|
+
* @returns The requested {@link Profile}.
|
|
29
30
|
*/
|
|
30
31
|
async getProfile(username) {
|
|
31
32
|
const res = await (0, profile_1.getProfile)(username, this.auth);
|
|
@@ -46,7 +47,7 @@ class Scraper {
|
|
|
46
47
|
* @param maxTweets The maximum number of tweets to return.
|
|
47
48
|
* @param includeReplies Whether or not replies should be included in the response.
|
|
48
49
|
* @param searchMode The category filter to apply to the search. Defaults to `Top`.
|
|
49
|
-
* @returns An
|
|
50
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filters.
|
|
50
51
|
*/
|
|
51
52
|
searchTweets(query, maxTweets, searchMode = search_1.SearchMode.Top) {
|
|
52
53
|
return (0, search_1.searchTweets)(query, maxTweets, searchMode, this.auth);
|
|
@@ -55,7 +56,7 @@ class Scraper {
|
|
|
55
56
|
* Fetches profiles from Twitter.
|
|
56
57
|
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
57
58
|
* @param maxProfiles The maximum number of profiles to return.
|
|
58
|
-
* @returns An
|
|
59
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filter(s).
|
|
59
60
|
*/
|
|
60
61
|
searchProfiles(query, maxProfiles) {
|
|
61
62
|
return (0, search_1.searchProfiles)(query, maxProfiles, this.auth);
|
|
@@ -92,34 +93,34 @@ class Scraper {
|
|
|
92
93
|
/**
|
|
93
94
|
* Fetches tweets from a Twitter user.
|
|
94
95
|
* @param user The user whose tweets should be returned.
|
|
95
|
-
* @param maxTweets The maximum number of tweets to return.
|
|
96
|
-
* @returns An
|
|
96
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
97
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
97
98
|
*/
|
|
98
|
-
getTweets(user, maxTweets) {
|
|
99
|
+
getTweets(user, maxTweets = 200) {
|
|
99
100
|
return (0, tweets_1.getTweets)(user, maxTweets, this.auth);
|
|
100
101
|
}
|
|
101
102
|
/**
|
|
102
103
|
* Fetches tweets from a Twitter user using their ID.
|
|
103
104
|
* @param userId The user whose tweets should be returned.
|
|
104
|
-
* @param maxTweets The maximum number of tweets to return.
|
|
105
|
-
* @returns An
|
|
105
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
106
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
106
107
|
*/
|
|
107
|
-
getTweetsByUserId(userId, maxTweets) {
|
|
108
|
+
getTweetsByUserId(userId, maxTweets = 200) {
|
|
108
109
|
return (0, tweets_1.getTweetsByUserId)(userId, maxTweets, this.auth);
|
|
109
110
|
}
|
|
110
111
|
/**
|
|
111
112
|
* Fetches the most recent tweet from a Twitter user.
|
|
112
113
|
* @param user The user whose latest tweet should be returned.
|
|
113
|
-
* @param includeRetweets Whether or not to include retweets.
|
|
114
|
+
* @param includeRetweets Whether or not to include retweets. Defaults to `false`.
|
|
114
115
|
* @returns The {@link Tweet} object or `null`/`undefined` if it couldn't be fetched.
|
|
115
116
|
*/
|
|
116
|
-
getLatestTweet(user, includeRetweets) {
|
|
117
|
+
getLatestTweet(user, includeRetweets = false) {
|
|
117
118
|
return (0, tweets_1.getLatestTweet)(user, includeRetweets, this.auth);
|
|
118
119
|
}
|
|
119
120
|
/**
|
|
120
121
|
* Fetches a single tweet.
|
|
121
122
|
* @param id The ID of the tweet to fetch.
|
|
122
|
-
* @returns The
|
|
123
|
+
* @returns The {@link Tweet} object, or `null` if it couldn't be fetched.
|
|
123
124
|
*/
|
|
124
125
|
getTweet(id) {
|
|
125
126
|
return (0, tweets_1.getTweet)(id, this.auth);
|
package/dist/scraper.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":";;;AACA,+BAAoE;AACpE,iCAAuD;AACvD,2CAA8C;AAC9C,uCAAuE;AACvE,qCAMkB;AAElB,qCAAqC;AACrC,qCAMkB;AAElB,MAAM,KAAK,GAAG,qBAAqB,CAAC;AAEpC;;;GAGG;AACH,MAAa,OAAO;IAIlB
|
|
1
|
+
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":";;;AACA,+BAAoE;AACpE,iCAAuD;AACvD,2CAA8C;AAC9C,uCAAuE;AACvE,qCAMkB;AAElB,qCAAqC;AACrC,qCAMkB;AAElB,MAAM,KAAK,GAAG,qBAAqB,CAAC;AAEpC;;;GAGG;AACH,MAAa,OAAO;IAIlB;;;;OAIG;IACH;QACE,IAAI,CAAC,IAAI,GAAG,IAAI,uBAAgB,CAAC,iBAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAgB,CAAC,kBAAY,CAAC,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU,CAAC,QAAgB;QACtC,MAAM,GAAG,GAAG,MAAM,IAAA,oBAAU,EAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,qBAAqB,CAAC,UAAkB;QACnD,MAAM,GAAG,GAAG,MAAM,IAAA,+BAAqB,EAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACI,YAAY,CACjB,KAAa,EACb,SAAiB,EACjB,aAAyB,mBAAU,CAAC,GAAG;QAEvC,OAAO,IAAA,qBAAY,EAAC,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,cAAc,CACnB,KAAa,EACb,WAAmB;QAEnB,OAAO,IAAA,uBAAc,EAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;;;;;;OAQG;IACI,iBAAiB,CACtB,KAAa,EACb,SAAiB,EACjB,UAAsB,EACtB,MAAe;QAEf,OAAO,IAAA,0BAAiB,EAAC,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5E,CAAC;IAED;;;;;;OAMG;IACI,mBAAmB,CACxB,KAAa,EACb,WAAmB,EACnB,MAAe;QAEf,OAAO,IAAA,4BAAmB,EAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACI,SAAS;QACd,OAAO,IAAA,kBAAS,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACI,SAAS,CAAC,IAAY,EAAE,SAAS,GAAG,GAAG;QAC5C,OAAO,IAAA,kBAAS,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;OAKG;IACI,iBAAiB,CACtB,MAAc,EACd,SAAS,GAAG,GAAG;QAEf,OAAO,IAAA,0BAAiB,EAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,CAAC;IAED;;;;;OAKG;IACI,cAAc,CACnB,IAAY,EACZ,eAAe,GAAG,KAAK;QAEvB,OAAO,IAAA,uBAAc,EAAC,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACI,QAAQ,CAAC,EAAU;QACxB,OAAO,IAAA,iBAAQ,EAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACI,aAAa;QAClB,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;IAC5D,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,UAAU;QACrB,OAAO,CACL,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,CACvE,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,KAAK,CAChB,QAAgB,EAChB,QAAgB,EAChB,KAAc;QAEd,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,2BAAe,CAAC,kBAAY,CAAC,CAAC;QACnD,MAAM,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,MAAM;QACjB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QAE/B,6CAA6C;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,uBAAgB,CAAC,iBAAW,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAgB,CAAC,kBAAY,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,UAAU;QACrB,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,UAAU,CAAC,OAA4B;QAClD,MAAM,QAAQ,GAAG,IAAI,2BAAe,CAAC,kBAAY,CAAC,CAAC;QACnD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC5B,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;SACrD;QAED,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,YAAY;QACvB,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,gBAAgB,EAAE,CAAC;QAC/C,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACvD,CAAC;IAED;;;;;OAKG;IACH,6DAA6D;IACtD,UAAU,CAAC,OAAe;QAC/B,OAAO,CAAC,IAAI,CACV,oIAAoI,CACrI,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,6DAA6D;IACtD,cAAc,CAAC,MAAc;QAClC,OAAO,CAAC,IAAI,CACV,uFAAuF,CACxF,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAI,GAAwB;QAChD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,MAAM,GAAG,CAAC,GAAG,CAAC;SACf;QAED,OAAO,GAAG,CAAC,KAAK,CAAC;IACnB,CAAC;CACF;AAvQD,0BAuQC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@the-convocation/twitter-scraper",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"main": "dist/_module.js",
|
|
5
5
|
"repository": "https://github.com/the-convocation/twitter-scraper.git",
|
|
6
6
|
"author": "karashiiro <49822414+karashiiro@users.noreply.github.com>",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"scripts": {
|
|
12
12
|
"build": "tsc",
|
|
13
13
|
"commit": "cz",
|
|
14
|
-
"docs:generate": "typedoc --
|
|
14
|
+
"docs:generate": "typedoc --options typedoc.json",
|
|
15
15
|
"docs:deploy": "yarn docs:generate && gh-pages -d docs",
|
|
16
16
|
"format": "prettier --write \\\"src/**/*.ts\\\" \\\"test/**/*.ts\\\"",
|
|
17
17
|
"prepare": "husky install",
|