instauto 9.1.1 → 9.1.4

package/README.md

@@ -25,7 +25,7 @@ Now there is a GUI application for those who don't want to code: [SimpleInstaBot
 
 You can run this code for example once every day using cron or pm2 or similar
 
-See [index.js](https://github.com/mifi/instauto/blob/master/index.js) for available options.
+See [index.js](https://github.com/mifi/instauto/blob/master/src/index.js) for available options.
 
 ## Supported functionality
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "instauto",
-  "version": "9.1.1",
+  "version": "9.1.4",
   "description": "Instagram automation library written in Node.js",
   "main": "src/index.js",
   "scripts": {

package/src/index.js

@@ -58,6 +58,7 @@ const Instauto = async (db, browser, options) => {
   } = options;
 
   let myUsername = myUsernameIn;
+  const userDataCache = {};
 
   assert(cookiesPath);
   assert(db);
@@ -73,7 +74,6 @@ const Instauto = async (db, browser, options) => {
 
   // State
   let page;
-  let graphqlUserMissing = false;
 
   async function takeScreenshot() {
     if (!screenshotOnError) return;
@@ -169,15 +169,20 @@ const Instauto = async (db, browser, options) => {
   }
 
   async function gotoWithRetry(url) {
+    const maxAttempts = 3;
     for (let attempt = 0; ; attempt += 1) {
       logger.log(`Goto ${url}`);
       const response = await page.goto(url);
-      await sleep(1000);
+      await sleep(2000);
       const status = response.status();
 
       // https://www.reddit.com/r/Instagram/comments/kwrt0s/error_560/
       // https://github.com/mifi/instauto/issues/60
-      if (![560, 429].includes(status) || attempt > 3) return status;
+      if (![560, 429].includes(status)) return status;
+
+      if (attempt > maxAttempts) {
+        throw new Error(`Navigate to user failed after ${maxAttempts} attempts, last status: ${status}`);
+      }
 
       logger.info(`Got ${status} - Retrying request later...`);
       if (status === 429) logger.warn('429 Too Many Requests could mean that Instagram suspects you\'re using a bot. You could try to use the Instagram Mobile app from the same IP for a few days first');
@@ -185,73 +190,78 @@ const Instauto = async (db, browser, options) => {
     }
   }
 
-  async function safeGotoUser(url, checkPageForUsername) {
-    const status = await gotoWithRetry(url);
-    if (status === 200) {
-      if (checkPageForUsername != null) {
-        // some pages return 200 but nothing there (I think deleted accounts)
-        // https://github.com/mifi/SimpleInstaBot/issues/48
-        // example: https://www.instagram.com/victorialarson__/
-        // so we check if the page has the user's name on it
-        return page.evaluate((username) => window.find(username), checkPageForUsername);
-      }
-      return true;
-    }
-    if (status === 404) {
-      logger.log('User not found');
-      return false;
-    }
-    throw new Error(`Navigate to user failed with status ${status}`);
+  const getUserPageUrl = (username) => `${instagramBaseUrl}/${encodeURIComponent(username)}`;
+
+  function isAlreadyOnUserPage(username) {
+    const url = getUserPageUrl(username);
+    // optimization: already on URL? (ignore trailing slash)
+    return (page.url().replace(/\/$/, '') === url.replace(/\/$/, ''));
   }
 
   async function navigateToUser(username) {
-    const url = `${instagramBaseUrl}/${encodeURIComponent(username)}`;
-    if (page.url().replace(/\/$/, '') === url.replace(/\/$/, '')) return true; // optimization: already on URL? (ignore trailing slash)
+    if (isAlreadyOnUserPage(username)) return true;
+
     // logger.log('navigating from', page.url(), 'to', url);
     logger.log(`Navigating to user ${username}`);
-    return safeGotoUser(url, username);
+
+    const url = getUserPageUrl(username);
+    const status = await gotoWithRetry(url);
+    if (status === 404) {
+      logger.warn('User page returned 404');
+      return false;
+    }
+
+    if (status === 200) {
+      // some pages return 200 but nothing there (I think deleted accounts)
+      // https://github.com/mifi/SimpleInstaBot/issues/48
+      // example: https://www.instagram.com/victorialarson__/
+      // so we check if the page has the user's name on it
+      const foundUsernameOnPage = await page.evaluate((u) => window.find(u), username);
+      if (!foundUsernameOnPage) logger.warn(`Cannot find "${username}" on page`);
+      return foundUsernameOnPage;
+    }
+
+    throw new Error(`Navigate to user failed with status ${status}`);
   }
 
   async function navigateToUserWithCheck(username) {
     if (!(await navigateToUser(username))) throw new Error('User not found');
   }
 
-  async function getPageJson() {
-    return JSON.parse(await (await (await page.$('pre')).getProperty('textContent')).jsonValue());
-  }
-
   async function navigateToUserAndGetData(username) {
-    // https://github.com/mifi/SimpleInstaBot/issues/36
-    if (graphqlUserMissing) {
-      // https://stackoverflow.com/questions/37593025/instagram-api-get-the-userid
-      // https://stackoverflow.com/questions/17373886/how-can-i-get-a-users-media-from-instagram-without-authenticating-as-a-user
-      const found = await safeGotoUser(`${instagramBaseUrl}/${encodeURIComponent(username)}?__a=1`);
-      if (!found) throw new Error('User not found');
+    const cachedUserData = userDataCache[username];
 
-      const json = await getPageJson();
-
-      const { user } = json.graphql;
+    if (isAlreadyOnUserPage(username)) {
+      // assume we have data
+      return cachedUserData;
+    }
 
+    if (cachedUserData != null) {
+      // if we already have userData, just navigate
       await navigateToUserWithCheck(username);
-      return user;
+      return cachedUserData;
     }
 
-    await navigateToUserWithCheck(username);
-
-    // eslint-disable-next-line no-underscore-dangle
-    const sharedData = await page.evaluate(() => window._sharedData);
-    try {
-      // eslint-disable-next-line prefer-destructuring
-      return sharedData.entry_data.ProfilePage[0].graphql.user;
+    // intercept special XHR network request that fetches user's data and store it in a cache
+    // TODO fallback to DOM to get user ID if this request fails?
+    // https://github.com/mifi/SimpleInstaBot/issues/125#issuecomment-1145354294
+    const [foundResponse] = await Promise.all([
+      page.waitForResponse((response) => {
+        const request = response.request();
+        return request.method() === 'GET' && new RegExp(`https:\\/\\/i\\.instagram\\.com\\/api\\/v1\\/users\\/web_profile_info\\/\\?username=${encodeURIComponent(username.toLowerCase())}`).test(request.url());
+      }),
+      navigateToUserWithCheck(username),
+      // page.waitForNavigation({ waitUntil: 'networkidle0' }),
+    ]);
+
+    const json = JSON.parse(await foundResponse.text());
+    const userData = json.data.user;
+    userDataCache[username] = userData;
+    return userData;
+  }
 
-      // JSON.parse(Array.from(document.getElementsByTagName('script')).find(el => el.innerHTML.startsWith('window.__additionalDataLoaded(\'feed\',')).innerHTML.replace(/^window.__additionalDataLoaded\('feed',({.*})\);$/, '$1'));
-      // JSON.parse(Array.from(document.getElementsByTagName('script')).find(el => el.innerHTML.startsWith('window._sharedData')).innerHTML.replace(/^window._sharedData ?= ?({.*});$/, '$1'));
-      // Array.from(document.getElementsByTagName('a')).find(el => el.attributes?.href?.value.includes(`${username}/followers`)).innerText
-    } catch (err) {
-      logger.warn('Missing graphql in page, falling back to alternative method...');
-      graphqlUserMissing = true; // Store as state so we don't have to do this every time from now on.
-      return navigateToUserAndGetData(username); // Now try again with alternative method
-    }
+  async function getPageJson() {
+    return JSON.parse(await (await (await page.$('pre')).getProperty('textContent')).jsonValue());
   }
 
   async function isActionBlocked() {
@@ -321,7 +331,7 @@ const Instauto = async (db, browser, options) => {
   }
 
   async function followUser(username) {
-    await navigateToUserWithCheck(username);
+    await navigateToUserAndGetData(username);
     const elementHandle = await findFollowButton();
 
     if (!elementHandle) {
@@ -363,7 +373,7 @@ const Instauto = async (db, browser, options) => {
   // See https://github.com/timgrossmann/InstaPy/pull/2345
   // https://github.com/timgrossmann/InstaPy/issues/2355
   async function unfollowUser(username) {
-    await navigateToUserWithCheck(username);
+    await navigateToUserAndGetData(username);
     logger.log(`Unfollowing user ${username}`);
 
     const res = { username, time: new Date().getTime() };
@@ -562,7 +572,7 @@ const Instauto = async (db, browser, options) => {
   async function likeUserImages({ username, likeImagesMin, likeImagesMax } = {}) {
     if (!likeImagesMin || !likeImagesMax || likeImagesMax < likeImagesMin || likeImagesMin < 1) throw new Error('Invalid arguments');
 
-    await navigateToUserWithCheck(username);
+    await navigateToUserAndGetData(username);
 
     logger.log(`Liking ${likeImagesMin}-${likeImagesMax} user images`);
     try {
@@ -581,6 +591,7 @@ const Instauto = async (db, browser, options) => {
       logger.log('Skipping already followed user', username);
       return false;
     }
+
     const graphqlUser = await navigateToUserAndGetData(username);
 
     const followedByCount = graphqlUser.edge_followed_by.count;
@@ -632,9 +643,9 @@ const Instauto = async (db, browser, options) => {
 
     let numFollowedForThisUser = 0;
 
-    const userData = await navigateToUserAndGetData(username);
+    const { id: userId } = await navigateToUserAndGetData(username);
 
-    for await (const followersBatch of getFollowersOrFollowingGenerator({ userId: userData.id, getFollowers: true })) {
+    for await (const followersBatch of getFollowersOrFollowingGenerator({ userId, getFollowers: true })) {
       logger.log('User followers batch', followersBatch);
 
       for (const follower of followersBatch) {
@@ -646,13 +657,13 @@ const Instauto = async (db, browser, options) => {
             return;
           }
 
-          if (enableFollow) {
-            if (await followUserRespectingRestrictions({ username: follower, skipPrivate })) {
-              numFollowedForThisUser += 1;
-            }
-          }
+          let didActuallyFollow = false;
+          if (enableFollow) didActuallyFollow = await followUserRespectingRestrictions({ username: follower, skipPrivate });
+          if (didActuallyFollow) numFollowedForThisUser += 1;
 
-          if (enableLikeImages) {
+          const didFailToFollow = enableFollow && !didActuallyFollow;
+
+          if (enableLikeImages && !didFailToFollow) {
             // Note: throws error if user isPrivate
             await likeUserImages({ username: follower, likeImagesMin, likeImagesMax });
           }
@@ -706,6 +717,8 @@ const Instauto = async (db, browser, options) => {
             const userFound = await navigateToUser(username);
 
             if (!userFound) {
+              // to avoid repeatedly unfollowing failed users, flag them as already unfollowed
+              logger.log('User not found for unfollow');
               await addPrevUnfollowedUser({ username, time: new Date().getTime(), noActionTaken: true });
               await sleep(3000);
             } else {
@@ -740,6 +753,8 @@ const Instauto = async (db, browser, options) => {
       }
     }
 
+    logger.log('Done with unfollowing', i, j);
+
     return j;
   }
 
@@ -937,16 +952,14 @@ const Instauto = async (db, browser, options) => {
     throw new Error('Don\'t know what\'s my username');
   }
 
-  const myUserData = await navigateToUserAndGetData(myUsername);
-  const myUserId = myUserData.id;
+  const { id: myUserId } = await navigateToUserAndGetData(myUsername);
 
   // --- END OF INITIALIZATION
 
   async function doesUserFollowMe(username) {
     try {
       logger.info('Checking if user', username, 'follows us');
-      const userData = await navigateToUserAndGetData(username);
-      const userId = userData.id;
+      const { id: userId } = await navigateToUserAndGetData(username);
 
       const elementHandles = await page.$x("//a[contains(.,' following')][contains(@href,'/following')]");
       if (elementHandles.length === 0) throw new Error('Following button not found');