gologin 1.0.46 → 1.0.49

package/browser-user-data-manager.js

@@ -3,6 +3,7 @@ const os = require('os');
 const request = require('requestretry');
 const { rmdirSync, createWriteStream } = require('fs');
 const { access, readFile, writeFile, mkdir, readdir, copyFile } = require('fs').promises;
+const crypto = require('crypto');
 
 const fontsCollection = require('./fonts');
 
@@ -12,6 +13,7 @@ const FONTS_DIR_NAME = 'fonts';
 const HOMEDIR = os.homedir();
 const BROWSER_PATH = path.join(HOMEDIR, '.gologin', 'browser');
 const OS_PLATFORM = process.platform;
+const DEFAULT_ORBITA_EXTENSIONS_NAMES = ['Google Hangouts', 'Chromium PDF Viewer', 'CryptoTokenExtension', 'Web Store'];
 
 class BrowserUserDataManager {
   static downloadCookies({ profileId, ACCESS_TOKEN, API_BASE_URL }) {
@@ -114,7 +116,7 @@ class BrowserUserDataManager {
     await writeFile(path.join(profilePath, 'Default', 'fonts_config'), result);
   }
 
-  static setExtPaths(settings = {}, profileExtensionsCheckRes = []) {
+  static setExtPathsAndRemoveDeleted(settings = {}, profileExtensionsCheckRes = []) {
     const formattedLocalExtArray = profileExtensionsCheckRes.map((el) => {
       const [extFolderName = ''] = el.split(path.sep).reverse();
       const [originalId] = extFolderName.split('@');
@@ -128,21 +130,60 @@ class BrowserUserDataManager {
       }
     }).filter(Boolean);
 
-    if (!formattedLocalExtArray.length) {
-      return;
-    }
-
     const extensionsSettings = settings.extensions?.settings || {};
     const extensionsEntries = Object.entries(extensionsSettings);
 
     extensionsEntries.forEach((extensionObj) => {
-      const [extensionsId] = extensionObj;
-      const localExtObj = formattedLocalExtArray.find(el => el.originalId === extensionsId);
+      let [extensionId, currentExtSettings = {}] = extensionObj;
+      const extName = currentExtSettings.manifest?.name || '';
+      let extPath = currentExtSettings.path || '';
+      let originalId = '';
+
+      const isExtensionToBeDeleted = ['resources', 'passwords-ext', 'cookies-ext'].some(substring => extPath.includes(substring))
+        || DEFAULT_ORBITA_EXTENSIONS_NAMES.includes(extName);
+      if (isExtensionToBeDeleted) {
+        delete extensionsSettings[extensionId];
+        return;
+      }
+
+      if (os.platform() === 'win32') {
+        extPath = extPath.replace(/\//g, '\\');
+      } else {
+        extPath = extPath.replace(/\\/g, '/');
+      }
+      extensionsSettings[extensionId].path = extPath;
+
+      const isExtensionManageable = ['chrome-extensions', 'user-extensions'].some(substring => extPath.includes(substring));
+      if (isExtensionManageable) {
+        const [extFolderName] = extPath.split(path.sep).reverse();
+        [originalId] = extFolderName.split('@');
+        const isExtensionInProfileSettings = formattedLocalExtArray.find(el => el.path.includes(originalId));
+        if (!isExtensionInProfileSettings) {
+          delete extensionsSettings[extensionId];
+          return;
+        }
+
+        if (!currentExtSettings.manifest?.key) {
+          const hexEncodedPath = crypto.createHash('sha256').update(extPath).digest('hex');
+          const newId = hexEncodedPath.split('').slice(0, 32).map(symbol => extIdEncoding[symbol]).join('');
+          delete extensionsSettings[extensionId];
+
+          extensionsSettings[newId] = currentExtSettings;
+          extensionId = newId;
+        }
+      } else {
+        const splittedPath = extPath.split(path.sep);
+        if (splittedPath.length === 2) {
+          [originalId] = splittedPath
+        }
+      }
+
+      const localExtObj = originalId && formattedLocalExtArray.find(el => el.path.includes(originalId));
       if (!localExtObj) {
         return;
       }
 
-      extensionsSettings[extensionsId].path = localExtObj?.path || '';
+      extensionsSettings[extensionId].path = localExtObj?.path || '';
     });
 
     return extensionsSettings;
@@ -202,6 +243,25 @@ class BrowserUserDataManager {
   }
 }
 
+const extIdEncoding = {
+  0: 'a',
+  1: 'b',
+  2: 'c',
+  3: 'd',
+  4: 'e',
+  5: 'f',
+  6: 'g',
+  7: 'h',
+  8: 'i',
+  9: 'j',
+  a: 'k',
+  b: 'l',
+  c: 'm',
+  d: 'n',
+  e: 'o',
+  f: 'p',
+};
+
 module.exports = {
   BrowserUserDataManager,
 }

package/extensions-manager.js

@@ -249,6 +249,28 @@ class ExtensionsManager {
 
     return Promise.all(removeFoldersPromises);
   }
+
+  getExtensionsToInstall(extensionsFromPref, extensionsFromDB) {
+    if (!extensionsFromPref) {
+      return [];
+    }
+
+    const objectEntries = Object.entries(extensionsFromPref);
+    const extensionsInPref = objectEntries?.map(([_, settings]) => {
+      const [extFolderName] = settings.path.split(path.sep).reverse();
+      const [originalId] = extFolderName.split('@');
+      return originalId;
+    }) || [];
+
+    return extensionsFromDB.reduce((acc, extension) => {
+      const [extFolderName] = extension.split(path.sep).reverse();
+      const [originalId] = extFolderName.split('@');
+      if (!extensionsInPref.includes(originalId)) {
+        acc.push(extension);
+      }
+      return acc;
+    }, []);
+  }
 }
 
 const crxToZip = (buf) => {

package/gologin.js

@@ -50,6 +50,7 @@ class GoLogin {
     this.writeCookesFromServer = options.writeCookesFromServer;
     this.remote_debugging_port = options.remote_debugging_port || 0;
     this.timezone = options.timezone;
+    this.extensionPathsToInstall = [];
 
     if (options.tmpdir) {
       this.tmpdir = options.tmpdir;
@@ -59,16 +60,16 @@ class GoLogin {
       }
     }
 
-    this.cookiesFilePath = path.join(this.tmpdir, `gologin_profile_${this.profile_id}`, 'Default', 'Cookies');
+    this.cookiesFilePath = path.join(this.tmpdir, `gologin_profile_${this.profile_id}`, 'Default', 'Network', 'Cookies');
     this.profile_zip_path = path.join(this.tmpdir, `gologin_${this.profile_id}.zip`);
     debug('INIT GOLOGIN', this.profile_id);
   }
-  
+
   async checkBrowser() { return this.browserChecker.checkBrowser(this.autoUpdateBrowser) }
 
   async setProfileId(profile_id) {
     this.profile_id = profile_id;
-    this.cookiesFilePath = path.join(this.tmpdir, `gologin_profile_${this.profile_id}`, 'Default', 'Cookies');
+    this.cookiesFilePath = path.join(this.tmpdir, `gologin_profile_${this.profile_id}`, 'Default', 'Network', 'Cookies');
     this.profile_zip_path = path.join(this.tmpdir, `gologin_${this.profile_id}.zip`);
   }
 
@@ -124,15 +125,15 @@ class GoLogin {
   		}
   	})
     debug("profileResponse", profileResponse.statusCode, profileResponse.body);
-    
-    
+
+
     if (profileResponse.statusCode === 404) {
       throw new Error(JSON.parse(profileResponse.body).message);
-    }   
+    }
 
     if (profileResponse.statusCode === 403) {
       throw new Error(JSON.parse(profileResponse.body).message);
-    }   
+    }
 
   	if (profileResponse.statusCode !== 200) {
   		throw new Error(`Gologin /browser/${id} response error ${profileResponse.statusCode} INVALID TOKEN OR PROFILE NOT FOUND`);
@@ -140,7 +141,7 @@ class GoLogin {
 
     if (profileResponse.statusCode === 401) {
       throw new Error("invalid token");
-    }    
+    }
 
 
   	return JSON.parse(profileResponse.body);
@@ -254,14 +255,30 @@ class GoLogin {
     if (_.get(preferences, 'isM1')) {
       preferences.is_m1 = _.get(preferences, 'isM1');
     }
-    
+
+    if (_.get(preferences, 'os') == 'android'){
+      const devicePixelRatio = _.get(preferences, "devicePixelRatio");
+      const deviceScaleFactorCeil = Math.ceil(devicePixelRatio || 3.5);
+      let deviceScaleFactor = devicePixelRatio;
+      if (deviceScaleFactorCeil === devicePixelRatio) {
+        deviceScaleFactor += 0.00000001;
+      }
+
+      preferences.mobile = {
+        enable: true,
+        width: parseInt(this.resolution.width, 10),
+        height: parseInt(this.resolution.height, 10),
+        device_scale_factor: deviceScaleFactor,
+      }
+    }
+
     preferences.mediaDevices = {
       enable: preferences.mediaDevices.enableMasking,
       videoInputs: preferences.mediaDevices.videoInputs,
       audioInputs: preferences.mediaDevices.audioInputs,
       audioOutputs: preferences.mediaDevices.audioOutputs,
     }
-    
+
     return preferences;
   }
 
@@ -289,7 +306,7 @@ class GoLogin {
     });
 
     debug('createBrowserExtension done');
-  } 
+  }
 
   extractProfile(path, zipfile) {
     debug(`extactProfile ${zipfile}, ${path}`);
@@ -358,7 +375,7 @@ class GoLogin {
     } catch(e) {
       console.trace(e);
       profile_folder = await this.emptyProfileFolder();
-      await writeFile(this.profile_zip_path, profile_folder);      
+      await writeFile(this.profile_zip_path, profile_folder);
       await this.extractProfile(profilePath, this.profile_zip_path);
     }
 
@@ -377,12 +394,12 @@ class GoLogin {
     if (!prefFileExists) {
       debug('Preferences file not exists waiting', pref_file_name, '. Using empty profile');
       profile_folder = await this.emptyProfileFolder();
-      await writeFile(this.profile_zip_path, profile_folder);      
+      await writeFile(this.profile_zip_path, profile_folder);
       await this.extractProfile(profilePath, this.profile_zip_path);
     }
 
     const preferences_raw = await readFile(pref_file_name);
-    let preferences = JSON.parse(preferences_raw.toString());    
+    let preferences = JSON.parse(preferences_raw.toString());
     let proxy = _.get(profile, 'proxy');
     let name = _.get(profile, 'name');
     const chromeExtensions = _.get(profile, 'chromeExtensions');
@@ -406,13 +423,16 @@ class GoLogin {
       }
 
       let extSettings;
-      if (ExtensionsManagerInst.useLocalExtStorage && profileExtensionsCheckRes.length) {
-        extSettings = BrowserUserDataManager.setExtPaths(preferences, profileExtensionsCheckRes);
-      } else if (!ExtensionsManagerInst.useLocalExtStorage) {
+      if (ExtensionsManagerInst.useLocalExtStorage) {
+        extSettings = BrowserUserDataManager.setExtPathsAndRemoveDeleted(preferences, profileExtensionsCheckRes);
+      } else {
         const originalExtensionsFolder = path.join(profilePath, 'Default', 'Extensions');
         extSettings = await BrowserUserDataManager.setOriginalExtPaths(preferences, originalExtensionsFolder);
       }
 
+      this.extensionPathsToInstall =
+        ExtensionsManagerInst.getExtensionsToInstall(extSettings, profileExtensionsCheckRes);
+
       if (extSettings) {
         const currentExtSettings = preferences.extensions || {};
         currentExtSettings.settings = extSettings
@@ -433,7 +453,7 @@ class GoLogin {
         'username': _.get(profile, 'autoProxyUsername'),
         'password': _.get(profile, 'autoProxyPassword'),
       }
-        
+
       profile.proxy.username = _.get(profile, 'autoProxyUsername');
       profile.proxy.password = _.get(profile, 'autoProxyPassword');
     }
@@ -470,7 +490,7 @@ class GoLogin {
       publicIP: _.get(profile, 'webRTC.fillBasedOnIp') ? this._tz.ip : _.get(profile, 'webRTC.publicIp'),
       localIps: _.get(profile, 'webRTC.localIps', []),
     };
-    
+
     debug('profile.webRtc=', profile.webRtc);
     debug('profile.timezone=', profile.timezone);
     debug('profile.mediaDevices=', profile.mediaDevices);
@@ -526,11 +546,11 @@ class GoLogin {
     }
 
     const [languages] = this.language.split(';');
-    
+
     if(preferences.gologin==null){
       preferences.gologin = {};
     }
-    
+
     preferences.gologin.langHeader = gologin.language;
     preferences.gologin.languages = languages;
     // debug("convertedPreferences=", preferences.gologin)
@@ -685,7 +705,7 @@ class GoLogin {
 
   async spawnArguments() {
     const profile_path = this.profilePath();
-    
+
     let proxy = this.proxy;
     proxy = `${proxy.mode}://${proxy.host}:${proxy.port}`;
 
@@ -715,10 +735,10 @@ class GoLogin {
     let remote_debugging_port = this.remote_debugging_port;
     if (!remote_debugging_port) {
       remote_debugging_port = await this.getRandomPort();
-    } 
-    
+    }
+
     const profile_path = this.profilePath();
-    
+
     let proxy = this.proxy;
     let proxy_host = '';
     if (proxy) {
@@ -727,7 +747,7 @@ class GoLogin {
     }
 
     this.port = remote_debugging_port;
-    
+
     const ORBITA_BROWSER = this.executablePath || this.browserChecker.getOrbitaPath;
     console.log("ORBITA_BROWSER=", ORBITA_BROWSER)
     const env = {};
@@ -757,12 +777,28 @@ class GoLogin {
 
       let params = [
         `--remote-debugging-port=${remote_debugging_port}`,
-        `--user-data-dir=${profile_path}`, 
-        `--password-store=basic`, 
+        `--user-data-dir=${profile_path}`,
+        `--password-store=basic`,
         `--tz=${tz}`,
         `--lang=${browserLang}`,
       ];
 
+      if (this.extensionPathsToInstall.length) {
+        if (Array.isArray(this.extra_params) && this.extra_params.length) {
+          this.extra_params.forEach((param, index) => {
+            if (!param.includes('--load-extension=')) {
+              return;
+            }
+
+            const [_, extPathsString] = param.split('=');
+            const extPathsArray = extPathsString.split(',');
+            this.extensionPathsToInstall = [...this.extensionPathsToInstall, ...extPathsArray];
+            this.extra_params.splice(index, 1);
+          });
+        }
+        params.push(`--load-extension=${this.extensionPathsToInstall.join(',')}`);
+      }
+
       if (this.fontsMasking) {
         let arg = '--font-masking-mode=2';
         if (this.differentOs) {
@@ -788,13 +824,13 @@ class GoLogin {
       const child = execFile(ORBITA_BROWSER, params, {env});
       // const child = spawn(ORBITA_BROWSER, params, { env, shell: true });
       child.stdout.on('data', (data) => debug(data.toString()));
-      debug('SPAWN CMD', ORBITA_BROWSER, params.join(" "));      
+      debug('SPAWN CMD', ORBITA_BROWSER, params.join(" "));
     }
 
     debug('GETTING WS URL FROM BROWSER');
 
     let data = await requests.get(`http://127.0.0.1:${remote_debugging_port}/json/version`, {json: true});
-    
+
     debug('WS IS', _.get(data, 'body.webSocketDebuggerUrl', ''))
     this.is_active = true;
 
@@ -837,7 +873,7 @@ class GoLogin {
 
     if (!local) {
       await rimraf(path.join(this.tmpdir, `gologin_${this.profile_id}.zip`));
-    }    
+    }
     debug(`PROFILE ${this.profile_id} STOPPED AND CLEAR`);
     return false;
   }
@@ -847,7 +883,7 @@ class GoLogin {
     if (!this.port) {
       throw new Error('Empty GoLogin port');
     }
-    const ls = await spawn('fuser', 
+    const ls = await spawn('fuser',
       [
         '-k TERM',
         `-n tcp ${this.port}`
@@ -882,7 +918,7 @@ class GoLogin {
     await Promise.all(remove_dirs.map(d => {
       const path_to_remove = `${that.profilePath()}${d}`
       return new Promise(resolve => {
-        debug('DROPPING', path_to_remove);        
+        debug('DROPPING', path_to_remove);
         rimraf(path_to_remove, { maxBusyTries: 100 }, (e) => {
           // debug('DROPPING RESULT', e);
           resolve();
@@ -934,7 +970,7 @@ class GoLogin {
   		return false;
   	}
     debug('profile is', profileResponse.body);
-  	return true;  	
+  	return true;
   }
 
 
@@ -943,16 +979,16 @@ class GoLogin {
 
     if (options.os) {
       os = options.os;
-    } 
+    }
 
-    let fingerprint = await requests.get(`https://api.gologin.com/browser/fingerprint?os=${os}`,{
+    let fingerprint = await requests.get(`${API_URL}/browser/fingerprint?os=${os}`,{
       headers: {
         'Authorization': `Bearer ${this.access_token}`,
         'User-Agent': 'gologin-api',
       }
-    });    
+    });
 
-    return JSON.parse(fingerprint.body);    
+    return JSON.parse(fingerprint.body);
   }
 
   async create(options) {
@@ -960,7 +996,7 @@ class GoLogin {
 
     const fingerprint = await this.getRandomFingerprint(options);
     debug("fingerprint=", fingerprint)
-    
+
     if (fingerprint.statusCode === 500) {
       throw new Error("no valid random fingerprint check os param");
     }
@@ -1032,7 +1068,7 @@ class GoLogin {
   async update(options) {
     this.profile_id = options.id;
     const profile = await this.getProfile();
-    
+
     if (options.navigator) {
       Object.keys(options.navigator).map((e)=>{profile.navigator[e]=options.navigator[e]});
     }
@@ -1045,7 +1081,7 @@ class GoLogin {
       headers: {
         'Authorization': `Bearer ${this.access_token}`
       }
-    });    
+    });
     debug('response', JSON.stringify(response.body));
     return response.body
   }
@@ -1070,7 +1106,7 @@ class GoLogin {
       accuracy: profileGeolocationParams.accuracy,
     }
   };
-  
+
   getViewPort() {
     return { ...this.resolution };
   };
@@ -1150,7 +1186,7 @@ class GoLogin {
     if (!this.executablePath) {
      await this.checkBrowser();
     }
-    
+
     const ORBITA_BROWSER = this.executablePath || this.browserChecker.getOrbitaPath;
 
     const orbitaBrowserExists = await access(ORBITA_BROWSER).then(() => true).catch(() => false);
@@ -1222,7 +1258,7 @@ class GoLogin {
       return {'status': 'failure', 'code':  profileResponse.statusCode};
     }
     */
-    
+
     // if (profileResponse.body === 'ok') {
     const profile = await this.getProfile();
 
@@ -1231,7 +1267,7 @@ class GoLogin {
         'Authorization': `Bearer ${this.access_token}`
       }
     });
-    
+
     debug('profileResponse', profileResponse.statusCode, profileResponse.body);
 
     if (profileResponse.statusCode === 401){
@@ -1262,7 +1298,7 @@ class GoLogin {
     let wsUrl = await this.waitDebuggingUrl(delay_ms);
     if(wsUrl!=''){
       return { 'status': 'success', wsUrl }
-    } 
+    }
 
     return { 'status': 'failure', 'message': profileResponse.body };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin",
-  "version": "1.0.46",
+  "version": "1.0.49",
   "description": "A high-level API to control Orbita browser over GoLogin API",
   "main": "./gologin.js",
   "repository": {