@the-convocation/twitter-scraper 0.1.4 → 0.2.0

package/dist/tweets.js

@@ -1,65 +1,73 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getTweet = exports.getTweets = exports.fetchTweets = void 0;
+exports.getTweet = exports.getLatestTweet = exports.getTweetsByUserId = exports.getTweets = exports.fetchTweets = void 0;
 const api_1 = require("./api");
 const profile_1 = require("./profile");
 const timeline_1 = require("./timeline");
 const timeline_async_1 = require("./timeline-async");
-function fetchTweets(user, maxTweets, includeReplies, cursor, auth) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (maxTweets > 200) {
-            maxTweets = 200;
-        }
-        const userIdRes = yield (0, profile_1.getUserIdByScreenName)(user, auth);
+async function fetchTweets(userId, maxTweets, includeReplies, cursor, auth) {
+    if (maxTweets > 200) {
+        maxTweets = 200;
+    }
+    const params = new URLSearchParams();
+    (0, api_1.addApiParams)(params, includeReplies);
+    params.set('count', `${maxTweets}`);
+    params.set('userId', userId);
+    if (cursor != null && cursor != '') {
+        params.set('cursor', cursor);
+    }
+    const res = await (0, api_1.requestApi)(`https://api.twitter.com/2/timeline/profile/${userId}.json?${params.toString()}`, auth);
+    if (!res.success) {
+        throw res.err;
+    }
+    return (0, timeline_1.parseTweets)(res.value);
+}
+exports.fetchTweets = fetchTweets;
+function getTweets(user, maxTweets, includeReplies, auth) {
+    return (0, timeline_async_1.getTweetTimeline)(user, maxTweets, async (q, mt, c) => {
+        const userIdRes = await (0, profile_1.getUserIdByScreenName)(q, auth);
         if (!userIdRes.success) {
             throw userIdRes.err;
         }
-        const params = new URLSearchParams();
-        (0, api_1.addApiParams)(params, includeReplies);
-        params.set('count', `${maxTweets}`);
-        params.set('userId', userIdRes.value);
-        if (cursor != null && cursor != '') {
-            params.set('cursor', cursor);
-        }
-        const res = yield (0, api_1.requestApi)(`https://api.twitter.com/2/timeline/profile/${userIdRes.value}.json?${params.toString()}`, auth);
-        if (!res.success) {
-            throw res.err;
-        }
-        return (0, timeline_1.parseTweets)(res.value);
+        const { value: userId } = userIdRes;
+        return fetchTweets(userId, mt, includeReplies, c, auth);
     });
 }
-exports.fetchTweets = fetchTweets;
-function getTweets(user, maxTweets, includeReplies, auth) {
-    return (0, timeline_async_1.getTweetTimeline)(user, maxTweets, (q, mt, c) => {
+exports.getTweets = getTweets;
+function getTweetsByUserId(userId, maxTweets, includeReplies, auth) {
+    return (0, timeline_async_1.getTweetTimeline)(userId, maxTweets, (q, mt, c) => {
         return fetchTweets(q, mt, includeReplies, c, auth);
     });
 }
-exports.getTweets = getTweets;
-function getTweet(id, includeReplies, auth) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const params = new URLSearchParams();
-        (0, api_1.addApiParams)(params, includeReplies);
-        const res = yield (0, api_1.requestApi)(`https://twitter.com/i/api/2/timeline/conversation/${id}.json?${params.toString()}`, auth);
-        if (!res.success) {
-            throw res.err;
+exports.getTweetsByUserId = getTweetsByUserId;
+async function getLatestTweet(user, includeReplies, includeRetweets, auth) {
+    const max = includeRetweets ? 1 : 200;
+    const timeline = await getTweets(user, max, includeReplies, auth);
+    if (max == 1) {
+        return (await timeline.next()).value;
+    }
+    for await (const tweet of timeline) {
+        if (!tweet.isRetweet) {
+            return tweet;
         }
-        const { tweets } = (0, timeline_1.parseTweets)(res.value);
-        for (const tweet of tweets) {
-            if (tweet.id === id) {
-                return tweet;
-            }
+    }
+    return null;
+}
+exports.getLatestTweet = getLatestTweet;
+async function getTweet(id, includeReplies, auth) {
+    const params = new URLSearchParams();
+    (0, api_1.addApiParams)(params, includeReplies);
+    const res = await (0, api_1.requestApi)(`https://twitter.com/i/api/2/timeline/conversation/${id}.json?${params.toString()}`, auth);
+    if (!res.success) {
+        throw res.err;
+    }
+    const { tweets } = (0, timeline_1.parseTweets)(res.value);
+    for (const tweet of tweets) {
+        if (tweet.id === id) {
+            return tweet;
         }
-        return null;
-    });
+    }
+    return null;
 }
 exports.getTweet = getTweet;
 //# sourceMappingURL=tweets.js.map

package/dist/tweets.js.map

@@ -1 +1 @@
-{"version":3,"file":"tweets.js","sourceRoot":"","sources":["../src/tweets.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAAiD;AAEjD,uCAAkD;AAClD,yCAA2E;AAC3E,qDAAoD;AAmDpD,SAAsB,WAAW,CAC/B,IAAY,EACZ,SAAiB,EACjB,cAAuB,EACvB,MAA0B,EAC1B,IAAsB;;QAEtB,IAAI,SAAS,GAAG,GAAG,EAAE;YACnB,SAAS,GAAG,GAAG,CAAC;SACjB;QAED,MAAM,SAAS,GAAG,MAAM,IAAA,+BAAqB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,MAAM,SAAS,CAAC,GAAG,CAAC;SACrB;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAErC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;SAC9B;QAED,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,8CACE,SAAS,CAAC,KACZ,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EAC5B,IAAI,CACL,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,MAAM,GAAG,CAAC,GAAG,CAAC;SACf;QAED,OAAO,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CAAA;AApCD,kCAoCC;AAED,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,cAAuB,EACvB,IAAsB;IAEtB,OAAO,IAAA,iCAAgB,EAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QACpD,OAAO,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AATD,8BASC;AAED,SAAsB,QAAQ,CAC5B,EAAU,EACV,cAAuB,EACvB,IAAsB;;QAEtB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,qDAAqD,EAAE,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EACnF,IAAI,CACL,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;YAChB,MAAM,GAAG,CAAC,GAAG,CAAC;SACf;QAED,MAAM,EAAE,MAAM,EAAE,GAAG,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC1B,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;gBACnB,OAAO,KAAK,CAAC;aACd;SACF;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CAAA;AAxBD,4BAwBC"}
+{"version":3,"file":"tweets.js","sourceRoot":"","sources":["../src/tweets.ts"],"names":[],"mappings":";;;AAAA,+BAAiD;AAEjD,uCAAkD;AAClD,yCAA2E;AAC3E,qDAAoD;AAiE7C,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,MAA0B,EAC1B,IAAiB;IAEjB,IAAI,SAAS,GAAG,GAAG,EAAE;QACnB,SAAS,GAAG,GAAG,CAAC;KACjB;IAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;IACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAErC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7B,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,EAAE,EAAE;QAClC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;KAC9B;IAED,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,8CAA8C,MAAM,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EAChF,IAAI,CACL,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;QAChB,MAAM,GAAG,CAAC,GAAG,CAAC;KACf;IAED,OAAO,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAChC,CAAC;AA7BD,kCA6BC;AAED,SAAgB,SAAS,CACvB,IAAY,EACZ,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QAC1D,MAAM,SAAS,GAAG,MAAM,IAAA,+BAAqB,EAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACtB,MAAM,SAAS,CAAC,GAAG,CAAC;SACrB;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAEpC,OAAO,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC;AAhBD,8BAgBC;AAED,SAAgB,iBAAiB,CAC/B,MAAc,EACd,SAAiB,EACjB,cAAuB,EACvB,IAAiB;IAEjB,OAAO,IAAA,iCAAgB,EAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;QACtD,OAAO,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC;AATD,8CASC;AAEM,KAAK,UAAU,cAAc,CAClC,IAAY,EACZ,cAAuB,EACvB,eAAwB,EACxB,IAAiB;IAEjB,MAAM,GAAG,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;IAElE,IAAI,GAAG,IAAI,CAAC,EAAE;QACZ,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;KACtC;IAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE;QAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;YACpB,OAAO,KAAK,CAAC;SACd;KACF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AApBD,wCAoBC;AAEM,KAAK,UAAU,QAAQ,CAC5B,EAAU,EACV,cAAuB,EACvB,IAAiB;IAEjB,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;IACrC,IAAA,kBAAY,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAErC,MAAM,GAAG,GAAG,MAAM,IAAA,gBAAU,EAC1B,qDAAqD,EAAE,SAAS,MAAM,CAAC,QAAQ,EAAE,EAAE,EACnF,IAAI,CACL,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE;QAChB,MAAM,GAAG,CAAC,GAAG,CAAC;KACf;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,IAAA,sBAAW,EAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,IAAI,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;YACnB,OAAO,KAAK,CAAC;SACd;KACF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAxBD,4BAwBC"}

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "@the-convocation/twitter-scraper",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "main": "dist/_module.js",
   "repository": "https://github.com/the-convocation/twitter-scraper.git",
   "author": "karashiiro <49822414+karashiiro@users.noreply.github.com>",
   "license": "MIT",
+  "engines": {
+    "node": ">=16"
+  },
   "scripts": {
     "build": "tsc",
     "commit": "cz",
@@ -15,27 +18,32 @@
     "test": "jest"
   },
   "dependencies": {
-    "got": "^12.6.0",
-    "got-scraping": "^3.2.13",
-    "tslib": "^2.5.0"
+    "cross-fetch": "^4.0.0-alpha.5",
+    "headers-polyfill": "^3.1.2",
+    "set-cookie-parser": "^2.6.0",
+    "tough-cookie": "^4.1.2",
+    "tslib": "^2.5.2"
   },
   "devDependencies": {
-    "@commitlint/cli": "^17.6.1",
-    "@commitlint/config-conventional": "^17.6.1",
-    "@types/jest": "^29.5.0",
-    "@typescript-eslint/eslint-plugin": "^5.58.0",
-    "@typescript-eslint/parser": "^5.58.0",
+    "@commitlint/cli": "^17.6.3",
+    "@commitlint/config-conventional": "^17.6.3",
+    "@tsconfig/node16": "^16.1.0",
+    "@types/jest": "^29.5.1",
+    "@types/set-cookie-parser": "^2.4.2",
+    "@types/tough-cookie": "^4.0.2",
+    "@typescript-eslint/eslint-plugin": "^5.59.7",
+    "@typescript-eslint/parser": "^5.59.7",
     "cz-conventional-changelog": "^3.3.0",
-    "eslint": "^8.38.0",
+    "eslint": "^8.41.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
     "gh-pages": "^5.0.0",
     "husky": "^8.0.3",
     "jest": "^29.5.0",
-    "lint-staged": "^13.2.1",
-    "prettier": "^2.8.7",
+    "lint-staged": "^13.2.2",
+    "prettier": "^2.8.8",
     "ts-jest": "^29.1.0",
-    "typedoc": "^0.24.1",
+    "typedoc": "^0.24.7",
     "typescript": "^5.0.4"
   },
   "config": {