@the-convocation/twitter-scraper 0.1.5 → 0.2.0

package/dist/tweets.js

@@ -1,55 +1,37 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getTweet = exports.getLatestTweet = exports.getTweetsByUserId = exports.getTweets = exports.fetchTweets = void 0;
 const api_1 = require("./api");
 const profile_1 = require("./profile");
 const timeline_1 = require("./timeline");
 const timeline_async_1 = require("./timeline-async");
-function fetchTweets(userId, maxTweets, includeReplies, cursor, auth) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (maxTweets > 200) {
-            maxTweets = 200;
-        }
-        const params = new URLSearchParams();
-        (0, api_1.addApiParams)(params, includeReplies);
-        params.set('count', `${maxTweets}`);
-        params.set('userId', userId);
-        if (cursor != null && cursor != '') {
-            params.set('cursor', cursor);
-        }
-        const res = yield (0, api_1.requestApi)(`https://api.twitter.com/2/timeline/profile/${userId}.json?${params.toString()}`, auth);
-        if (!res.success) {
-            throw res.err;
-        }
-        return (0, timeline_1.parseTweets)(res.value);
-    });
+async function fetchTweets(userId, maxTweets, includeReplies, cursor, auth) {
+    if (maxTweets > 200) {
+        maxTweets = 200;
+    }
+    const params = new URLSearchParams();
+    (0, api_1.addApiParams)(params, includeReplies);
+    params.set('count', `${maxTweets}`);
+    params.set('userId', userId);
+    if (cursor != null && cursor != '') {
+        params.set('cursor', cursor);
+    }
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/2/timeline/profile/${userId}.json?${params.toString()}`, auth);
+    if (!res.success) {
+        throw res.err;
+    }
+    return (0, timeline_1.parseTweets)(res.value);
 }
 exports.fetchTweets = fetchTweets;
 function getTweets(user, maxTweets, includeReplies, auth) {
-    return (0, timeline_async_1.getTweetTimeline)(user, maxTweets, (q, mt, c) => __awaiter(this, void 0, void 0, function* () {
-        const userIdRes = yield (0, profile_1.getUserIdByScreenName)(q, auth);
+    return (0, timeline_async_1.getTweetTimeline)(user, maxTweets, async (q, mt, c) => {
+        const userIdRes = await (0, profile_1.getUserIdByScreenName)(q, auth);
         if (!userIdRes.success) {
             throw userIdRes.err;
         }
         const { value: userId } = userIdRes;
         return fetchTweets(userId, mt, includeReplies, c, auth);
-    }));
+    });
 }
 exports.getTweets = getTweets;
 function getTweetsByUserId(userId, maxTweets, includeReplies, auth) {
@@ -58,56 +40,34 @@ function getTweetsByUserId(userId, maxTweets, includeReplies, auth) {
     });
 }
 exports.getTweetsByUserId = getTweetsByUserId;
-function getLatestTweet(user, includeReplies, includeRetweets, auth) {
-    var _a, e_1, _b, _c;
-    return __awaiter(this, void 0, void 0, function* () {
-        const max = includeRetweets ? 1 : 200;
-        const timeline = yield getTweets(user, max, includeReplies, auth);
-        if (max == 1) {
-            return (yield timeline.next()).value;
+async function getLatestTweet(user, includeReplies, includeRetweets, auth) {
+    const max = includeRetweets ? 1 : 200;
+    const timeline = await getTweets(user, max, includeReplies, auth);
+    if (max == 1) {
+        return (await timeline.next()).value;
+    }
+    for await (const tweet of timeline) {
+        if (!tweet.isRetweet) {
+            return tweet;
         }
-        try {
-            for (var _d = true, timeline_2 = __asyncValues(timeline), timeline_2_1; timeline_2_1 = yield timeline_2.next(), _a = timeline_2_1.done, !_a;) {
-                _c = timeline_2_1.value;
-                _d = false;
-                try {
-                    const tweet = _c;
-                    if (!tweet.isRetweet) {
-                        return tweet;
-                    }
-                }
-                finally {
-                    _d = true;
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = timeline_2.return)) yield _b.call(timeline_2);
-            }
-            finally { if (e_1) throw e_1.error; }
-        }
-        return null;
-    });
+    }
+    return null;
 }
 exports.getLatestTweet = getLatestTweet;
-function getTweet(id, includeReplies, auth) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const params = new URLSearchParams();
-        (0, api_1.addApiParams)(params, includeReplies);
-        const res = yield (0, api_1.requestApi)(`https://twitter.com/i/api/2/timeline/conversation/${id}.json?${params.toString()}`, auth);
-        if (!res.success) {
-            throw res.err;
+async function getTweet(id, includeReplies, auth) {
+    const params = new URLSearchParams();
+    (0, api_1.addApiParams)(params, includeReplies);
+    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/2/timeline/conversation/${id}.json?${params.toString()}`, auth);
+    if (!res.success) {
+        throw res.err;
+    }
+    const { tweets } = (0, timeline_1.parseTweets)(res.value);
+    for (const tweet of tweets) {
+        if (tweet.id === id) {
+            return tweet;
         }
-        const { tweets } = (0, timeline_1.parseTweets)(res.value);
-        for (const tweet of tweets) {
-            if (tweet.id === id) {
-                return tweet;
-            }
-        }
-        return null;
-    });
+    }
+    return null;
 }
 exports.getTweet = getTweet;
 //# sourceMappingURL=tweets.js.map

package/dist/tweets.js.map

@@ -1 +1 @@
-{"version":3,"file":"tweets.js","sourceRoot":"","sources":["../src/tweets.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,+BAAiD;AAEjD,uCAAkD;AAClD,yCAA2E;AAC3E,qDAAoD;AAiEpD,SAAsB,WAAW,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,MAA0B,EAC1B,IAAiB;;QAEjB,IAAI,SAAS,GAAG,GAAG,EAAE;YACnB,SAAS,GAAG,GAAG,CAAC;SACjB;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAErC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7B,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;SAC9B;QAED,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,8CAA8C,MAAM,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EAChF,IAAI,CACL,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,MAAM,GAAG,CAAC,GAAG,CAAC;SACf;QAED,OAAO,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CAAA;AA7BD,kCA6BC;AAED,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,IAAI,EAAE,SAAS,EAAE,CAAO,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QAC1D,MAAM,SAAS,GAAG,MAAM,IAAA,+BAAqB,EAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,MAAM,SAAS,CAAC,GAAG,CAAC;SACrB;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAEpC,OAAO,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAA,CAAC,CAAC;AACL,CAAC;AAhBD,8BAgBC;AAED,SAAgB,iBAAiB,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QACtD,OAAO,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AATD,8CASC;AAED,SAAsB,cAAc,CAClC,IAAY,EACZ,cAAuB,EACvB,eAAwB,EACxB,IAAiB;;;QAEjB,MAAM,GAAG,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QAElE,IAAI,GAAG,IAAI,CAAC,EAAE;YACZ,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;SACtC;;YAED,KAA0B,eAAA,aAAA,cAAA,QAAQ,CAAA,cAAA;gBAAR,wBAAQ;gBAAR,WAAQ;;oBAAvB,MAAM,KAAK,KAAA,CAAA;oBACpB,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;wBACpB,OAAO,KAAK,CAAC;qBACd;;;;;aACF;;;;;;;;;QAED,OAAO,IAAI,CAAC;;CACb;AApBD,wCAoBC;AAED,SAAsB,QAAQ,CAC5B,EAAU,EACV,cAAuB,EACvB,IAAiB;;QAEjB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,qDAAqD,EAAE,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EACnF,IAAI,CACL,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,MAAM,GAAG,CAAC,GAAG,CAAC;SACf;QAED,MAAM,EAAE,MAAM,EAAE,GAAG,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;gBACnB,OAAO,KAAK,CAAC;aACd;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CAAA;AAxBD,4BAwBC"}
+{"version":3,"file":"tweets.js","sourceRoot":"","sources":["../src/tweets.ts"],"names":[],"mappings":";;;AAAA,+BAAiD;AAEjD,uCAAkD;AAClD,yCAA2E;AAC3E,qDAAoD;AAiE7C,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,MAA0B,EAC1B,IAAiB;IAEjB,IAAI,SAAS,GAAG,GAAG,EAAE;QACnB,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;IACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAErC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7B,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,EAAE,EAAE;QAClC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;KAC9B;IAED,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,8CAA8C,MAAM,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EAChF,IAAI,CACL,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;QAChB,MAAM,GAAG,CAAC,GAAG,CAAC;KACf;IAED,OAAO,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AA7BD,kCA6BC;AAED,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QAC1D,MAAM,SAAS,GAAG,MAAM,IAAA,+BAAqB,EAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,MAAM,SAAS,CAAC,GAAG,CAAC;SACrB;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAEpC,OAAO,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC;AAhBD,8BAgBC;AAED,SAAgB,iBAAiB,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QACtD,OAAO,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AATD,8CASC;AAEM,KAAK,UAAU,cAAc,CAClC,IAAY,EACZ,cAAuB,EACvB,eAAwB,EACxB,IAAiB;IAEjB,MAAM,GAAG,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;IAElE,IAAI,GAAG,IAAI,CAAC,EAAE;QACZ,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;KACtC;IAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE;QAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YACpB,OAAO,KAAK,CAAC;SACd;KACF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AApBD,wCAoBC;AAEM,KAAK,UAAU,QAAQ,CAC5B,EAAU,EACV,cAAuB,EACvB,IAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;IACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,qDAAqD,EAAE,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EACnF,IAAI,CACL,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;QAChB,MAAM,GAAG,CAAC,GAAG,CAAC;KACf;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;YACnB,OAAO,KAAK,CAAC;SACd;KACF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAxBD,4BAwBC"}

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "@the-convocation/twitter-scraper",
-  "version": "0.1.5",
+  "version": "0.2.0",
   "main": "dist/_module.js",
   "repository": "https://github.com/the-convocation/twitter-scraper.git",
   "author": "karashiiro <49822414+karashiiro@users.noreply.github.com>",
   "license": "MIT",
+  "engines": {
+    "node": ">=16"
+  },
   "scripts": {
     "build": "tsc",
     "commit": "cz",
@@ -15,14 +18,18 @@
     "test": "jest"
   },
   "dependencies": {
-    "got-scraping": "^3.2.13",
+    "cross-fetch": "^4.0.0-alpha.5",
+    "headers-polyfill": "^3.1.2",
+    "set-cookie-parser": "^2.6.0",
     "tough-cookie": "^4.1.2",
     "tslib": "^2.5.2"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.6.3",
     "@commitlint/config-conventional": "^17.6.3",
+    "@tsconfig/node16": "^16.1.0",
     "@types/jest": "^29.5.1",
+    "@types/set-cookie-parser": "^2.4.2",
     "@types/tough-cookie": "^4.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.7",
     "@typescript-eslint/parser": "^5.59.7",