instauto 9.1.9 → 9.1.10

package/README.md

@@ -2,10 +2,9 @@
 
 [![Discord](https://img.shields.io/discord/986052713425027072)](https://discord.gg/Rh3KT9zyhj) [![PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://paypal.me/mifino/usd)
 
-instauto is an Instagram automation/bot library written in modern, clean javascript using Google's Puppeteer. Goal is to be very easy to set up, use, and extend, and obey instagram's limits. Heavily inspired by [InstaPy](https://github.com/timgrossmann/InstaPy), but I thought it was way too heavy and hard to setup.
+instauto is an Instagram automation/bot library (API) written in modern, clean javascript using Google's Puppeteer. Goal is to be very easy to set up, use, and extend, and obey instagram's limits. Heavily inspired by [InstaPy](https://github.com/timgrossmann/InstaPy), but I thought it was way too heavy and hard to setup.
 
-**NEW! 🎉**
-Now there is a GUI application for those who don't want to code: [SimpleInstaBot](https://mifi.github.io/SimpleInstaBot/)
+There is also a GUI application for those who don't want to code: [SimpleInstaBot](https://mifi.github.io/SimpleInstaBot/)
 
 
 ## Setup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "instauto",
-  "version": "9.1.9",
+  "version": "9.1.10",
   "description": "Instagram automation library written in Node.js",
   "main": "src/index.js",
   "scripts": {

package/src/index.js

@@ -16,6 +16,15 @@ function shuffleArray(arrayIn) {
   return array;
 }
 
+// https://stackoverflow.com/questions/14822153/escape-single-quote-in-xpath-with-nokogiri
+// example str: "That's mine", he said.
+function escapeXpathStr(str) {
+  const parts = str.split("'").map((token) => `'${token}'`);
+  if (parts.length === 1) return `${parts[0]}`;
+  const str2 = parts.join(', "\'", ');
+  return `concat(${str2})`;
+}
+
 const botWorkShiftHours = 16;
 
 const dayMs = 24 * 60 * 60 * 1000;
@@ -122,7 +131,7 @@ const Instauto = async (db, browser, options) => {
   const sleep = (ms, deviation = 1) => {
     let msWithDev = ((Math.random() * deviation) + 1) * ms;
     if (dryRun) msWithDev = Math.min(3000, msWithDev); // for dryRun, no need to wait so long
-    logger.log('Waiting', Math.round(msWithDev / 1000), 'sec');
+    logger.log('Waiting', (msWithDev / 1000).toFixed(2), 'sec');
     return new Promise(resolve => setTimeout(resolve, msWithDev));
   };
 
@@ -178,8 +187,9 @@ const Instauto = async (db, browser, options) => {
     for (let attempt = 0; ; attempt += 1) {
       logger.log(`Goto ${url}`);
       const response = await gotoUrl(url);
-      await sleep(2000);
       const status = response.status();
+      logger.log('Page loaded');
+      await sleep(2000);
 
       // https://www.reddit.com/r/Instagram/comments/kwrt0s/error_560/
       // https://github.com/mifi/instauto/issues/60
@@ -217,12 +227,14 @@ const Instauto = async (db, browser, options) => {
     }
 
     if (status === 200) {
+      // logger.log('Page returned 200 ☑️');
       // some pages return 200 but nothing there (I think deleted accounts)
       // https://github.com/mifi/SimpleInstaBot/issues/48
       // example: https://www.instagram.com/victorialarson__/
       // so we check if the page has the user's name on it
-      const foundUsernameOnPage = await page.evaluate((u) => window.find(u), username);
-      if (!foundUsernameOnPage) logger.warn(`Cannot find "${username}" on page`);
+      const elementHandles = await page.$x(`//body//main//*[contains(text(),${escapeXpathStr(username)})]`);
+      const foundUsernameOnPage = elementHandles.length > 0;
+      if (!foundUsernameOnPage) logger.warn(`Cannot find text "${username}" on page`);
       return foundUsernameOnPage;
     }
 
@@ -247,22 +259,85 @@ const Instauto = async (db, browser, options) => {
       return cachedUserData;
     }
 
+    async function getUserDataFromPage() {
+      // https://github.com/mifi/instauto/issues/115#issuecomment-1199335650
+      // to test in browser: document.getElementsByTagName('html')[0].innerHTML.split('\n');
+      try {
+        const body = await page.content();
+        for (let q of body.split(/\r?\n/)) {
+          if (q.includes('edge_followed_by')) {
+            // eslint-disable-next-line prefer-destructuring
+            q = q.split(',[],[')[1];
+            // eslint-disable-next-line prefer-destructuring
+            q = q.split(']]]')[0];
+            q = JSON.parse(q);
+            // eslint-disable-next-line no-underscore-dangle
+            q = q.data.__bbox.result.response;
+            q = q.replace(/\\/g, '');
+            q = JSON.parse(q);
+            return q.data.user;
+          }
+        }
+      } catch (err) {
+        logger.warn('Failed to get user data from page', err);
+      }
+      return undefined;
+    }
+
     // intercept special XHR network request that fetches user's data and store it in a cache
     // TODO fallback to DOM to get user ID if this request fails?
     // https://github.com/mifi/SimpleInstaBot/issues/125#issuecomment-1145354294
-    const [foundResponse] = await Promise.all([
-      page.waitForResponse((response) => {
-        const request = response.request();
-        return request.method() === 'GET' && new RegExp(`https:\\/\\/i\\.instagram\\.com\\/api\\/v1\\/users\\/web_profile_info\\/\\?username=${encodeURIComponent(username.toLowerCase())}`).test(request.url());
-      }),
-      navigateToUserWithCheck(username),
-      // page.waitForNavigation({ waitUntil: 'networkidle0' }),
-    ]);
-
-    const json = JSON.parse(await foundResponse.text());
-    const userData = json.data.user;
-    userDataCache[username] = userData;
-    return userData;
+    async function getUserDataFromInterceptedRequest() {
+      const t = setTimeout(async () => {
+        logger.log('Unable to intercept request, will send manually');
+        try {
+          await page.evaluate(async (username2) => {
+            const response = await window.fetch(`https://i.instagram.com/api/v1/users/web_profile_info/?username=${encodeURIComponent(username2.toLowerCase())}`, { mode: 'cors', credentials: 'include', headers: { 'x-ig-app-id': '936619743392459' } });
+            await response.json(); // else it will not finish the request
+          }, username);
+          // todo `https://i.instagram.com/api/v1/users/${userId}/info/`
+          // https://www.javafixing.com/2022/07/fixed-can-get-instagram-profile-picture.html?m=1
+        } catch (err) {
+          logger.error('Failed to manually send request', err);
+        }
+      }, 5000);
+
+      try {
+        const [foundResponse] = await Promise.all([
+          page.waitForResponse((response) => {
+            const request = response.request();
+            return request.method() === 'GET' && new RegExp(`https:\\/\\/i\\.instagram\\.com\\/api\\/v1\\/users\\/web_profile_info\\/\\?username=${encodeURIComponent(username.toLowerCase())}`).test(request.url());
+          }, { timeout: 30000 }),
+          navigateToUserWithCheck(username),
+          // page.waitForNavigation({ waitUntil: 'networkidle0' }),
+        ]);
+
+        const json = JSON.parse(await foundResponse.text());
+        return json.data.user;
+      } finally {
+        clearTimeout(t);
+      }
+    }
+
+    logger.log('Trying to get user data from HTML');
+
+    await navigateToUserWithCheck(username);
+    let userData = await getUserDataFromPage();
+    if (userData) {
+      userDataCache[username] = userData;
+      return userData;
+    }
+
+    logger.log('Need to intercept network request to get user data');
+
+    // works for old accounts only:
+    userData = await getUserDataFromInterceptedRequest();
+    if (userData) {
+      userDataCache[username] = userData;
+      return userData;
+    }
+
+    return undefined;
   }
 
   async function getPageJson() {
@@ -1031,7 +1106,7 @@ const Instauto = async (db, browser, options) => {
       return followsMe === false;
     }
 
-    await safelyUnfollowUserList(allFollowingGenerator, limit, condition);
+    return safelyUnfollowUserList(allFollowingGenerator, limit, condition);
   }
 
   async function unfollowAllUnknown({ limit } = {}) {
@@ -1048,7 +1123,7 @@ const Instauto = async (db, browser, options) => {
       return true;
     }
 
-    await safelyUnfollowUserList(unfollowUsersGenerator, limit, condition);
+    return safelyUnfollowUserList(unfollowUsersGenerator, limit, condition);
   }
 
   async function unfollowOldFollowed({ ageInDays, limit } = {}) {