gologin 2.0.1 → 2.0.3

package/README.md

@@ -23,11 +23,11 @@ To have an access to the page below you need <a href="https://app.gologin.com/#/
 ### Example
 
 ```js
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 (async () => {
   const GL = new GoLogin({
@@ -128,11 +128,11 @@ stop current browser without removing archived profile
 ### example-local-profile.js
 
 ```js
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 (async () => {
   const GL = new GoLogin({

package/browser-user-data-manager.js

@@ -62,7 +62,7 @@ export const downloadFonts = async (fontsList = [], profilePath) => {
   const files = await readdir(browserFontsPath);
   const fontsToDownload = fontsList.filter(font => !files.includes(font));
 
-  let promises = fontsToDownload.map(font => get(FONTS_URL + font, {
+  let promises = fontsToDownload.map(font => requestretry.get(FONTS_URL + font, {
     maxAttempts: 5,
     retryDelay: 2000,
     timeout: 30 * 1000,

package/cookies-manager.js

@@ -1,6 +1,6 @@
 import { open } from 'sqlite';
-import pkg from 'sqlite3';
-const { Database, OPEN_READONLY } = pkg;
+import sqlite3 from 'sqlite3';
+const { Database, OPEN_READONLY } = sqlite3;
 
 const MAX_SQLITE_VARIABLES = 76;
 

package/examples/example-amazon-goless.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 (async () => {
   const GL = new GoLogin({

package/examples/example-amazon-headless.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 const delay = (time) => new Promise((resolve) => setTimeout(resolve, time));
 

package/examples/example-amazon.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 (async () => {
   const GL = new GoLogin({

package/examples/example-gmail.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 const delay = ms => new Promise(res => setTimeout(res, ms));
 

package/examples/example-local-profile.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 (async () => {
   const GL = new GoLogin({

package/examples/example-login-walmart.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 const delay = ms => new Promise(res => setTimeout(res, ms));
 

package/examples/example-timezone.js

@@ -1,8 +1,8 @@
-import pkg from 'puppeteer-core';
+import puppeteer from 'puppeteer-core';
 
 import GoLogin from './gologin.js';
 
-const { connect } = pkg;
+const { connect } = puppeteer;
 
 const delay = ms => new Promise(res => setTimeout(res, ms));
 

package/gologin.js

@@ -4,25 +4,25 @@ import decompress from 'decompress';
 import decompressUnzip from 'decompress-unzip';
 import { existsSync, mkdirSync, promises as _promises } from 'fs';
 import { get as _get } from 'https';
-import pkg from 'lodash';
+import lodash from 'lodash';
 import { tmpdir } from 'os';
 import { join, resolve as _resolve,sep } from 'path';
 import requests from 'requestretry';
 import rimraf from 'rimraf';
 import ProxyAgent from 'simple-proxy-agent';
 import util from 'util';
-import zipdir from 'zip-dir';
 
 import BrowserChecker from './browser-checker.js';
 import { composeFonts, downloadCookies, setExtPathsAndRemoveDeleted, setOriginalExtPaths, uploadCookies } from './browser-user-data-manager.js';
 import { getChunckedInsertValues, getDB, loadCookiesFromFile } from './cookies-manager.js';
 import ExtensionsManager from './extensions-manager.js';
 import { fontsCollection } from './fonts.js';
+import { archiveProfile } from './profile-archiver.js';
 
 const exec = util.promisify(execNonPromise);
 
 const { access, unlink, writeFile, readFile } = _promises;
-const { get, merge } = pkg;
+const { get, merge } = lodash;
 
 const SEPARATOR = sep;
 const API_URL = 'https://api.gologin.com';
@@ -442,7 +442,7 @@ export class GoLogin {
 
               return { profileExtensionsCheckRes: [] };
             }),
-          ExtensionsManagerInst.checkLocalUserChromeExtensions(userChromeExtensions)
+          ExtensionsManagerInst.checkLocalUserChromeExtensions(userChromeExtensions, this.profile_id)
             .then(res => ({ profileUserExtensionsCheckRes: res }))
             .catch((error) => {
               console.log('checkUserChromeExtensions error: ', error);
@@ -1014,20 +1014,7 @@ export class GoLogin {
     debug('profile sanitized');
 
     const profilePath = this.profilePath();
-    const fileBuff = await new Promise((resolve, reject) => zipdir(profilePath,
-      {
-        saveTo: zipPath,
-        filter: (path) => !/RunningChromeVersion/.test(path),
-      }, (err, buffer) => {
-        if (err) {
-          reject(err);
-
-          return;
-        }
-
-        resolve(buffer);
-      }),
-    );
+    const fileBuff = await archiveProfile(profilePath);
 
     debug('PROFILE ZIP CREATED', profilePath, zipPath);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin",
-  "version": "2.0.1",
+  "version": "2.0.3",
   "description": "A high-level API to control Orbita browser over GoLogin API",
   "main": "./gologin.js",
   "repository": {
@@ -15,6 +15,7 @@
   "license": "GPL-3.0",
   "dependencies": {
     "archiver": "^3.1.1",
+    "adm-zip": "^0.5.1",
     "child_process": "^1.0.2",
     "decompress": "^4.2.1",
     "decompress-unzip": "^4.0.1",
@@ -27,8 +28,7 @@
     "rimraf": "^3.0.2",
     "simple-proxy-agent": "^1.1.0",
     "sqlite": "^4.0.23",
-    "sqlite3": "^5.0.2",
-    "zip-dir": "^2.0.0"
+    "sqlite3": "^5.0.2"
   },
   "bugs": {
     "url": "https://github.com/gologinapp/gologin/issues"

package/profile-archiver.js

@@ -0,0 +1,81 @@
+import AdmZip from 'adm-zip';
+import { promises as _promises } from 'fs';
+import path from 'path';
+
+import { getDirectoriesForArchiver } from './profile-directories-to-remove.js';
+
+const { access } = _promises;
+
+export const archiveProfile = async (profileFolder = '', tryAgain = true) => {
+  const folderExists = await access(profileFolder).then(() => true, () => false);
+  if (!folderExists) {
+    throw new Error('Invalid profile folder path: ' + profileFolder);
+  }
+
+  const archive = new AdmZip();
+  archive.addLocalFolder(path.join(profileFolder, 'Default'), 'Default');
+  archive.addLocalFile(path.join(profileFolder, 'First Run'));
+  const dirsToRemove = getDirectoriesForArchiver();
+  dirsToRemove.forEach(entry => archive.deleteFile(entry));
+
+  const archiveIsValid = checkProfileArchiveIsValid(archive);
+  if (tryAgain && !archiveIsValid) {
+    await new Promise(r => setTimeout(() => r(), 300));
+
+    return archiveProfile(profileFolder, false);
+  }
+
+  return new Promise((resolve, reject) => archive.toBuffer(resolve, reject));
+};
+
+export const decompressProfile = async (zipPath = '', profileFolder = '') => {
+  const zipExists = await access(zipPath).then(() => true, () => false);
+  if (!zipExists) {
+    throw new Error('Invalid zip path: ' + zipPath);
+  }
+
+  const archive = new AdmZip(zipPath);
+  archive
+    .getEntries()
+    .forEach((elem) => {
+      if (
+        !elem.isDirectory &&
+        (
+          elem.entryName.includes('RunningChromeVersion') ||
+          elem.entryName.includes('SingletonLock') ||
+          elem.entryName.includes('SingletonSocket') ||
+          elem.entryName.includes('SingletonCookie')
+        )
+      ) {
+        archive.deleteFile(elem);
+      }
+    });
+
+  archive.extractAllTo(profileFolder, true);
+};
+
+export const checkProfileArchiveIsValid = (zipObject) => {
+  if (!zipObject) {
+    throw new Error('No zip object provided');
+  }
+
+  return zipObject
+    .getEntries()
+    .map(elem => {
+      if (elem.isDirectory) {
+        return false;
+      }
+
+      return elem.entryName.includes('Preferences') || elem.entryName.includes('Cookies');
+    })
+    .filter(Boolean)
+    .length >= 2;
+};
+
+const flatArray = (array = []) => array.map((elem) => {
+  if (Array.isArray(elem)) {
+    return flatArray(elem).flat();
+  }
+
+  return elem;
+}).flat().filter(Boolean);

package/profile-directories-to-remove.js

@@ -0,0 +1,55 @@
+const DEFAULT_FOLDER_USELESS_FILE = [
+  {
+    name: 'Cache',
+    subs: [],
+    isDirectory: true,
+  },
+  {
+    name: 'fonts_config',
+    subs: [],
+    isDirectory: false,
+  },
+  {
+    name: 'Service Worker',
+    subs: ['CacheStorage'],
+    isDirectory: true,
+  },
+  {
+    name: 'Code Cache',
+    subs: [],
+    isDirectory: true,
+  },
+  {
+    name: 'GPUCache',
+    subs: [],
+    isDirectory: true,
+  },
+];
+
+export const getDirectoriesToDeleteForNode = (routerSlash = '/') =>
+  DEFAULT_FOLDER_USELESS_FILE.reduce((res, el) => {
+    const basePath = routerSlash + 'Default' + routerSlash + el.name;
+    if (el.subs.length) {
+      el.subs.forEach(sub => res.push(basePath + routerSlash + sub));
+    } else {
+      res.push(basePath);
+    }
+
+    return res;
+  }, []);
+
+export const getDirectoriesForArchiver = () => DEFAULT_FOLDER_USELESS_FILE.reduce((res, el) => {
+  const { name, subs, isDirectory } = el;
+  const basePath = 'Default/' + name;
+
+  if (subs.length) {
+    subs.forEach((sub) => {
+      const resPath = basePath + '/' + (isDirectory ? sub + '/' : sub);
+      res.push(resPath);
+    });
+  } else {
+    res.push(basePath + (isDirectory ? '/' : ''));
+  }
+
+  return res;
+}, []);

package/user-extensions-manager.js

@@ -1,15 +1,10 @@
 import { createWriteStream, promises as _promises } from 'fs';
 import { join, sep } from 'path';
 import request from 'requestretry';
-import zipdir from 'zip-dir';
 
 import { CHROME_EXTENSIONS_PATH, composeExtractionPromises, USER_EXTENSIONS_PATH } from './common.js';
-import { extractExtension } from './extensions-extractor.js';
 
-const { readdir, rmdir, readFile, stat, mkdir, copyFile } = _promises;
-
-const MAX_FILE_SIZE = 80 * 1024 * 1024;
-const MAX_FILE_SIZE_MB = MAX_FILE_SIZE / 1024 / 1024;
+const { readdir, readFile, stat, mkdir, copyFile } = _promises;
 
 export class UserExtensionsManager {
   #existedUserExtensions = [];
@@ -78,127 +73,7 @@ export class UserExtensionsManager {
     this.#existedUserExtensions = fileList;
   }
 
-  async addCustomExtension(pathToFiles) {
-    try {
-      const filesSize = await checkFileSizeSync(pathToFiles);
-      const isZip = pathToFiles.endsWith('.zip');
-
-      if (filesSize > MAX_FILE_SIZE) {
-        throw new Error(`The maximum file size is ${MAX_FILE_SIZE_MB}MB`);
-      }
-
-      const customId = this.generateExtensionId();
-
-      if (isZip) {
-        const pathToExtract = join(USER_EXTENSIONS_PATH, customId);
-        await extractExtension(pathToFiles, pathToExtract);
-        pathToFiles = pathToExtract;
-      }
-
-      let fileList = (await readdir(pathToFiles).catch(() => ['cantReadError']))
-        .filter(folderContent => folderContent !== '.DS_Store');
-
-      if (fileList.length === 1 && !fileList.includes('cantReadError')) {
-        const isFolder = (await stat(pathToFiles)).isDirectory();
-        if (isFolder) {
-          const [folderName] = fileList;
-          pathToFiles = join(pathToFiles, folderName);
-          fileList = await readdir(pathToFiles).catch(() => ['cantReadError']);
-        }
-      }
-
-      if (fileList.includes('cantReadError')) {
-        throw new Error('Can\'t access folder');
-      }
-
-      if (!fileList.includes('manifest.json')) {
-        if (isZip) {
-          rmdir(pathToFiles);
-        }
-
-        throw new Error('There is no manifest.json in the extension folder');
-      }
-
-      if (!isZip) {
-        const destPath = join(USER_EXTENSIONS_PATH, customId);
-        await copyFolder(pathToFiles, destPath).catch(() => {
-          throw new Error('Something went wrong coping your folder');
-        });
-      }
-
-      const [nameIconId] = await this.getExtensionsNameAndImage([customId], USER_EXTENSIONS_PATH);
-
-      if (!nameIconId) {
-        throw new Error('Something went wrong. Please try again later');
-      }
-
-      const dbResult = await request(`${this.#API_BASE_URL}/extensions/create_user_extension`, {
-        method: 'POST',
-        headers: {
-          Authorization: `Bearer ${this.#ACCESS_TOKEN}`,
-          'user-agent': this.#USER_AGENT,
-          'x-two-factor-token': this.#TWO_FA_KEY || '',
-        },
-        body: {
-          extensionInfo: nameIconId,
-        },
-        json: true,
-      });
-
-      // if success - there is no body
-      if (dbResult.body) {
-        throw new Error('Something went wrong inserting your data to database');
-      }
-
-      const fileBuffer = await zipdir(pathToFiles).catch(() => null);
-      if (!fileBuffer) {
-        throw new Error('Something went wrong. Please try again later');
-      }
-
-      const signedUrl = await request.get(`${this.#API_BASE_URL}/extensions/upload_url?extId=${customId}`, {
-        headers: {
-          Authorization: `Bearer ${this.#ACCESS_TOKEN}`,
-          'user-agent': this.#USER_AGENT,
-          'x-two-factor-token': this.#TWO_FA_KEY || '',
-        },
-        maxAttempts: 3,
-        retryDelay: 2000,
-        timeout: 10 * 1000,
-        fullResponse: false,
-      });
-
-      const uploadResponse = await request.put(signedUrl, {
-        headers: {
-          'Content-Type': 'application/zip',
-          'Content-Length': Buffer.byteLength(fileBuffer),
-        },
-        body: fileBuffer,
-        maxBodyLength: Infinity,
-        maxContentLength: Infinity,
-        maxAttempts: 3,
-        retryDelay: 2000,
-        timeout: 30 * 1000,
-        fullResponse: true,
-      });
-
-      // if success - there is no body, in case of error - there will be an error in the body
-      if (uploadResponse.body) {
-        throw new Error('Your extension is added locally but we couldn\'t upload it to the cloud');
-      }
-
-      return {
-        status: 'success',
-        message: nameIconId,
-      };
-    } catch (e) {
-      return {
-        status: 'error',
-        message: e.message,
-      };
-    }
-  }
-
-  checkLocalUserChromeExtensions = async (userChromeExtensions) => {
+  checkLocalUserChromeExtensions = async (userChromeExtensions, profileId) => {
     if (!userChromeExtensions.length) {
       return;
     }
@@ -213,6 +88,8 @@ export class UserExtensionsManager {
       },
       body: {
         existedUserChromeExtensions: this.#existedUserExtensions,
+        profileId,
+        userChromeExtensions,
       },
     }) || [];