gologin 1.0.29 → 1.0.33

package/README.md

@@ -18,7 +18,7 @@ for running example.js install puppeteer-core
 Where is token? API token is <a href="https://app.gologin.com/#/personalArea/TokenApi" target="_blank">here</a>.
 To have an access to the page below you need <a href="https://app.gologin.com/#/createUser" target="_blank">register</a> GoLogin account.
 
-![Token API in Settings](https://user-images.githubusercontent.com/62306291/78453427-53220100-769a-11ea-9465-0aae3ae602b7.jpg)
+![Token API in Settings](https://user-images.githubusercontent.com/12957968/146891933-c3b60b4d-c850-47a5-8adf-bc8c37372664.gif)
 
 ### Example
 

package/browser-checker.js

@@ -260,7 +260,8 @@ class BrowserChecker {
   async replaceBrowser() {
     console.log('Copy Orbita to target path');
     if (PLATFORM === 'darwin') {
-      await rmdir(path.join(this.#browserPath, 'Orbita-Browser.app'), { recursive: true });
+      await this.deleteDir(path.join(this.#browserPath, 'Orbita-Browser.app'));
+
       const files = await readdir(path.join(this.#browserPath, EXTRACTED_FOLDER));
       const promises = [];
       files.forEach((filename) => {
@@ -268,22 +269,25 @@ class BrowserChecker {
           promises.push(copyFile(path.join(this.#browserPath, EXTRACTED_FOLDER, filename), path.join(this.#browserPath, filename)));
         }
       });
+
       return Promise.all(promises);
-    } else {
-      await rmdir(path.join(this.#browserPath, 'orbita-browser'), { recursive: true });
-      await this.copyDir(
-        path.join(this.#browserPath, EXTRACTED_FOLDER, 'orbita-browser'),
-        path.join(this.#browserPath, 'orbita-browser')
-      );
     }
+
+    const targetBrowserPath = path.join(this.#browserPath, 'orbita-browser');
+    await this.deleteDir(targetBrowserPath);
+
+    await this.copyDir(
+      path.join(this.#browserPath, EXTRACTED_FOLDER, 'orbita-browser'),
+      targetBrowserPath
+    );
   }
   
   async deleteOldArchives(deleteCurrentBrowser = false) {
     if (deleteCurrentBrowser) {
-      return await rmdir(path.join(this.#browserPath), { recursive: true });
+      return this.deleteDir(path.join(this.#browserPath));
     }
 
-    await rmdir(path.join(this.#browserPath, EXTRACTED_FOLDER), { recursive: true });
+    await this.deleteDir(path.join(this.#browserPath, EXTRACTED_FOLDER));
     return readdir(this.#browserPath)
       .then((files) => {
         const promises = [];
@@ -355,6 +359,19 @@ class BrowserChecker {
   get getOrbitaPath() {
     return this.#executableFilePath;
   }
+
+  async deleteDir(path = '') {
+    if (!path) {
+      return;
+    }
+
+    const directoryExists = await access(path).then(() => true).catch(() => false);
+    if (!directoryExists) {
+      return;
+    }
+
+    return rmdir(path, { recursive: true });
+  }
 }
 
 module.exports = BrowserChecker;

package/examples/example-amazon-headless.js

@@ -1,7 +1,9 @@
 const puppeteer = require('puppeteer-core');
 const GoLogin = require('../gologin');
 
-(async () =>{
+const delay = (time) => new Promise((resolve) => setTimeout(resolve, time));
+
+(async () =>{    
     const GL = new GoLogin({
         token: 'yU0token',
         profile_id: 'yU0Pr0f1leiD',
@@ -14,6 +16,7 @@ const GoLogin = require('../gologin');
     });
 
     const page = await browser.newPage();
+    await delay(300);
 
     const viewPort = GL.getViewPort();
     await page.setViewport({ width: Math.round(viewPort.width * 0.994), height: Math.round(viewPort.height * 0.92) });

package/gologin.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const os = require('os');
 const util = require('util');
 const rimraf = util.promisify(require('rimraf'));
+const { access, unlink, writeFile, readFile } = require('fs').promises;
 const exec = util.promisify(require('child_process').exec);
 const { spawn, execFile } = require('child_process');
 const FormData = require('form-data');
@@ -12,7 +13,7 @@ const ProxyAgent = require('simple-proxy-agent');
 const decompress = require('decompress');
 const decompressUnzip = require('decompress-unzip');
 const path = require('path');
-const shell = require('shelljs');
+const zipdir = require('zip-dir');
 
 const BrowserChecker = require('./browser-checker');
 const { BrowserUserDataManager } = require('./browser-user-data-manager');
@@ -54,7 +55,7 @@ class GoLogin {
       this.tmpdir = options.tmpdir;
       if (!fs.existsSync(this.tmpdir)) {
         debug('making tmpdir', this.tmpdir);
-        shell.mkdir('-p', this.tmpdir);
+        fs.mkdirSync(this.tmpdir, { recursive: true })
       }
     }
 
@@ -127,7 +128,7 @@ class GoLogin {
   		throw new Error(`Gologin /browser/${id} response error ${profileResponse.statusCode} INVALID TOKEN OR PROFILE NOT FOUND`);
   	}
 
-    if(profileResponse.statusCode == 401){
+    if (profileResponse.statusCode === 401) {
       throw new Error("invalid token");
     }    
 
@@ -135,77 +136,86 @@ class GoLogin {
   }
 
   async emptyProfile() {
-  	return fs.readFileSync(path.resolve(__dirname, 'gologin_zeroprofile.b64')).toString();
+    return readFile(path.resolve(__dirname, 'gologin_zeroprofile.b64')).then(res => res.toString());
   }
 
   async getProfileS3(s3path) {
+    if (!s3path) {
+      throw new Error('s3path not found');
+    }
+
     const token = this.access_token;
     debug('getProfileS3 token=', token, 'profile=', this.profile_id, 's3path=', s3path);
-    if (s3path) { //загрузка профиля из публичного бакета s3 быстрее
-      const s3url = `https://gprofiles.gologin.com/${s3path}`.replace(/\s+/mg, '+');
-      debug('loading profile from public s3 bucket, url=', s3url);
-      const profileResponse = await requests.get(s3url, {
-        encoding: null
-      });
-      if (profileResponse.statusCode !== 200) {
-        debug(`Gologin S3 BUCKET ${s3url} response error ${profileResponse.statusCode}  - use empty`);
-        return '';
-      }
-      return Buffer.from(profileResponse.body);
-    }
 
-    debug('old-way loading profile');
-    const profileResponse = await requests.get(`${API_URL}/browser/${this.profile_id}/profile-s3`, {
-      headers: {
-        'Authorization': `Bearer ${token}`,
-      },
+    const s3url = `https://gprofiles.gologin.com/${s3path}`.replace(/\s+/mg, '+');
+    debug('loading profile from public s3 bucket, url=', s3url);
+    const profileResponse = await requests.get(s3url, {
       encoding: null
     });
+
     if (profileResponse.statusCode !== 200) {
-      debug(`Gologin /browser/${this.profile_id} response error ${profileResponse.statusCode}  - use empty`);
+      debug(`Gologin S3 BUCKET ${s3url} response error ${profileResponse.statusCode}  - use empty`);
       return '';
     }
+
     return Buffer.from(profileResponse.body);
   }
 
-  async postFile(fileName, fileBody) {
-    debug('POSTING FILE', fileBody.length);
-    const fd = new FormData();
-    const boundary = fd.getBoundary();
-    const body = Buffer.concat([
-      Buffer.from('--'),
-      Buffer.from(boundary),
-      Buffer.from('\r\n'),
-      Buffer.from(`Content-Disposition: form-data; name="profile"; filename="${fileName}"`),
-      Buffer.from('\r\n'),
-      Buffer.from('\r\n'),
-      Buffer.from(fileBody),
-      Buffer.from('\r\n'),
-      Buffer.from('--'),
-      Buffer.from(boundary),
-      Buffer.from('--'),
-      Buffer.from('\r\n')
-    ]);
-    await new Promise((resolve) => {
-      let options = {
-        method: 'PATCH',
-        headers: {
-          'Authorization': `Bearer ${this.access_token}`,
-          'Content-Type': 'multipart/form-data; boundary=' + boundary,
-          'Content-Length': body.length
-        },
-        body: body,
-        url: `${API_URL}/browser/${this.profile_id}/profile-s3`,
-      };
-      requests(_.merge(options, {}), () => {
-        resolve();
-      });
+  async postFile(fileName, fileBuff) {
+    debug('POSTING FILE', fileBuff.length);
+    debug('Getting signed URL for S3');
+    const apiUrl = `${API_URL}/browser/${this.profile_id}/storage-signature`;
+
+    const signedUrl = await requests.get(apiUrl, {
+      headers: {
+        Authorization: `Bearer ${this.access_token}`,
+        'user-agent': 'gologin-api',
+      },
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 10 * 1000,
+      fullResponse: false,
     });
+
+    const [uploadedProfileUrl] = signedUrl.split('?');
+
+    console.log('Uploading profile by signed URL to S3');
+    const bodyBufferBiteLength = Buffer.byteLength(fileBuff);
+    console.log('BUFFER SIZE', bodyBufferBiteLength);
+
+    await requests.put(signedUrl, {
+      headers: {
+        'Content-Type': 'application/zip',
+        'Content-Length': bodyBufferBiteLength,
+      },
+      body: fileBuff,
+      maxBodyLength: Infinity,
+      maxContentLength: Infinity,
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 30 * 1000,
+      fullResponse: false,
+    });
+
+    const uploadedProfileMetadata = await requests.head(uploadedProfileUrl, {
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 10 * 1000,
+      fullResponse: true,
+    });
+
+    const uploadedFileLength = +uploadedProfileMetadata.headers['content-length'];
+    if (uploadedFileLength !== bodyBufferBiteLength) {
+      console.log('Uploaded file is incorrect. Retry with China File size:', uploadedFileLength);
+      throw new Error('Uploaded file is incorrect. Retry with China File size: ' + uploadedFileLength);
+    }
+
+    console.log('Profile has been uploaded to S3 successfully');
   }
 
   async emptyProfileFolder() {
     debug('get emptyProfileFolder');
-    const profile = fs.readFileSync(path.resolve(__dirname, 'gologin_zeroprofile.zip'));
+    const profile = await readFile(path.resolve(__dirname, 'gologin_zeroprofile.zip'));
     debug('emptyProfileFolder LENGTH ::', profile.length);
     return profile;
   }
@@ -246,9 +256,8 @@ class GoLogin {
       .then(() => {
         debug('extraction done');
         debug('create uid.json');
-        fs.writeFileSync(path.join(extPath, 'uid.json'), JSON.stringify({uid: that.profile_id}, null, 2))
-        debug('uid.json created', fs.readFileSync(path.join(extPath, 'uid.json')).toString())
-        return extPath;
+        return writeFile(path.join(extPath, 'uid.json'), JSON.stringify({ uid: that.profile_id }, null, 2))
+          .then(() => extPath);
     })
       .catch(async (e) => {
         debug('orbita extension error', e);
@@ -299,19 +308,21 @@ class GoLogin {
       height: parseInt(screenHeight, 10),
     };
 
-    if (!(local && fs.existsSync(this.profile_zip_path))) {
+    const profileZipExists = await access(this.profile_zip_path).then(() => true).catch(() => false);
+    if (!(local && profileZipExists)) {
       try {
         profile_folder = await this.getProfileS3(_.get(profile, 's3Path', ''));
       }
       catch (e) {
         debug('Cannot get profile - using empty', e);
       }
+
       debug('FILE READY', this.profile_zip_path);
       if (!profile_folder.length) {
         profile_folder = await this.emptyProfileFolder();
       }
-        
-      fs.writeFileSync(this.profile_zip_path, profile_folder);
+
+      await writeFile(this.profile_zip_path, profile_folder);
 
       debug('PROFILE LENGTH', profile_folder.length);
     } else {
@@ -323,20 +334,23 @@ class GoLogin {
     await this.extractProfile(profilePath, this.profile_zip_path);
     debug('extraction done');
 
-    if (fs.existsSync(path.join(profilePath, 'SingletonLock'))) {
+    const singletonLockPath = path.join(profilePath, 'SingletonLock');
+    const singletonLockExists = await access(singletonLockPath).then(() => true).catch(() => false);
+    if (singletonLockExists) {
       debug('removing SingletonLock');
-      fs.unlinkSync(path.join(profilePath, 'SingletonLock'));
+      await unlink(singletonLockPath);
       debug('SingletonLock removed');
     }
 
     const pref_file_name = path.join(profilePath, 'Default', 'Preferences');
     debug('reading', pref_file_name);
 
-    if (!fs.existsSync(pref_file_name)) {
+    const prefFileExists = await access(pref_file_name).then(() => true).catch(() => false);
+    if (!prefFileExists) {
       debug('Preferences file not exists waiting', pref_file_name);
     }
 
-    const preferences_raw = fs.readFileSync(pref_file_name);
+    const preferences_raw = await readFile(pref_file_name);
     let preferences = JSON.parse(preferences_raw.toString());
     let proxy = _.get(profile, 'proxy');
     let name = _.get(profile, 'name');
@@ -359,6 +373,11 @@ class GoLogin {
       profile.proxy.password = _.get(profile, 'autoProxyPassword');
     }
     // console.log('proxy=', proxy);
+
+    if (proxy.mode === 'geolocation') {
+      proxy.mode = 'http';
+    }
+
     if (proxy.mode === 'none') {
       proxy = null;
     }
@@ -431,7 +450,7 @@ class GoLogin {
       await BrowserUserDataManager.composeFonts(families, profilePath, this.differentOs);
     }
 
-    fs.writeFileSync(path.join(profilePath, 'Default', 'Preferences'), JSON.stringify(_.merge(preferences, {
+    await writeFile(path.join(profilePath, 'Default', 'Preferences'), JSON.stringify(_.merge(preferences, {
       gologin
     })));
 
@@ -444,20 +463,23 @@ class GoLogin {
   }
 
   async commitProfile() {
-    const data = await this.getProfileDataToUpdate();
-    debug('begin updating', data.length);
-    if (!data.length) {
+    const dataBuff = await this.getProfileDataToUpdate();
+
+    debug('begin updating', dataBuff.length);
+    if (!dataBuff.length) {
       debug('WARN: profile zip data empty - SKIPPING PROFILE COMMIT');
 
       return;
     }
+
     try {
       debug('Patching profile');
-      await this.postFile('profile', data);
+      await this.postFile('profile', dataBuff);
     }
     catch (e) {
       debug('CANNOT COMMIT PROFILE', e);
     }
+
     debug('COMMIT COMPLETED');
   }
 
@@ -477,16 +499,16 @@ class GoLogin {
 
   async checkPortAvailable(port) {
     debug('CHECKING PORT AVAILABLE', port);
+
     try {
       const { stdout, stderr } = await exec(`lsof -i:${port}`);
-      if (
-        stdout && stdout.match(/LISTEN/gmi)
-      ) {
+      if (stdout && stdout.match(/LISTEN/gmi)) {
         debug(`PORT ${port} IS BUSY`)
         return false;
       }
-    } catch (e) { }
+    } catch (e) {}
     debug(`PORT ${port} IS OPEN`);
+
     return true;
   }
 
@@ -501,23 +523,24 @@ class GoLogin {
   }
 
   async getTimeZone(proxy) {
-    if(this.timezone){
+    debug('getting timeZone proxy=', proxy);
+    if (this.timezone) {
       debug('getTimeZone from options', this.timezone);
       this._tz = this.timezone;
       return this._tz.timezone;
     }
 
     let data = null;
-    if (proxy) {
+    if (proxy!==null && proxy.mode !== "none") {
       if (proxy.mode.includes('socks')) {
         return this.getTimezoneWithSocks(proxy);
       }
 
       const proxyUrl = `${proxy.mode}://${proxy.username}:${proxy.password}@${proxy.host}:${proxy.port}`;
       debug('getTimeZone start https://time.gologin.com', proxyUrl);
-      data = await requests.get('https://time.gologin.com', { proxy: proxyUrl, timeout: 10 * 1000, maxAttempts: 2 });
+      data = await requests.get('https://time.gologin.com', { proxy: proxyUrl, timeout: 20 * 1000, maxAttempts: 5 });
     } else {
-      data = await requests.get('https://time.gologin.com', { timeout: 10 * 1000, maxAttempts: 2 });
+      data = await requests.get('https://time.gologin.com', { timeout: 20 * 1000, maxAttempts: 5 });
     }
     debug('getTimeZone finish', data.body);
     this._tz = JSON.parse(data.body);
@@ -598,7 +621,7 @@ class GoLogin {
 
   async spawnBrowser() {
     let remote_debugging_port = this.remote_debugging_port;
-    if(!remote_debugging_port){
+    if (!remote_debugging_port) {
       remote_debugging_port = await this.getRandomPort();
     } 
     
@@ -700,7 +723,9 @@ class GoLogin {
     if (this.is_stopping) {
       return true;
     }
-    const is_posting = options.postings || false;
+    const is_posting = options.posting ||
+      options.postings || // backward compability
+      false;
 
     if (this.uploadCookiesToServer) {
       await this.uploadProfileCookiesToServer();
@@ -776,34 +801,31 @@ class GoLogin {
 
   async getProfileDataToUpdate() {
     const zipPath = path.join(this.tmpdir, `gologin_${this.profile_id}_upload.zip`);
-    try {
-      fs.unlinkSync(zipPath);
-    }
-    catch (e) {
+    const zipExists = await access(zipPath).then(() => true).catch(() => false);
+    if (zipExists) {
+      await unlink(zipPath);
     }
+
     await this.sanitizeProfile();
     debug('profile sanitized');
-    await new Promise(resolve => {
-      debug('begin zipping');
-      execFile(`cd ${this.profilePath()}; /usr/bin/zip`, [
-        `-r ${zipPath}`,
-        '*'
-      ], {
-        shell: true
-      }, () => {
-        debug('zipping done');
-        resolve();
-      });
-    });
-    debug('PROFILE ZIP CREATED', this.profilePath(), zipPath);
-    try {
-      const data = fs.readFileSync(zipPath);
-      return data;
-    }
-    catch (e) {
-      debug('saveprofile error', e);
-      return '';
-    }
+
+    const profilePath = this.profilePath();
+    const fileBuff = await new Promise((resolve, reject) => zipdir(profilePath,
+      {
+        saveTo: zipPath,
+        filter: (path) => !/RunningChromeVersion/.test(path),
+      }, (err, buffer) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        resolve(buffer);
+      })
+    )
+
+    debug('PROFILE ZIP CREATED', profilePath, zipPath);
+    return fileBuff;
   }
 
   async profileExists() {
@@ -847,11 +869,11 @@ class GoLogin {
     const fingerprint = await this.getRandomFingerprint(options);
     debug("fingerprint=", fingerprint)
     
-    if(fingerprint.statusCode == 500){
+    if (fingerprint.statusCode === 500) {
       throw new Error("no valid random fingerprint check os param");
     }
 
-    if(fingerprint.statusCode == 401){
+    if (fingerprint.statusCode === 401) {
       throw new Error("invalid token");
     }
 
@@ -881,7 +903,7 @@ class GoLogin {
     let user_agent = options.navigator?.userAgent;
     let orig_user_agent = json.navigator.userAgent;
     Object.keys(options).map((e)=>{ json[e] = options[e] });
-    if(user_agent=='random'){
+    if (user_agent === 'random') {
       json.navigator.userAgent = orig_user_agent;
     }
     // console.log('profileOptions', json);
@@ -894,11 +916,11 @@ class GoLogin {
       json,
     });
 
-    if(response.body.statusCode==400){
+    if (response.body.statusCode === 400) {
       throw new Error(`gologin failed account creation with status code, ${data.statusCode} DATA  ${JSON.stringify(response.body.message)}`);
     }
 
-    if(response.body.statusCode==500){
+    if (response.body.statusCode === 500) {
       throw new Error(`gologin failed account creation with status code, ${data.statusCode}`);
     }
     debug(JSON.stringify(response.body));
@@ -1036,6 +1058,13 @@ class GoLogin {
     if (!this.executablePath) {
      await this.checkBrowser();
     }
+    
+    const ORBITA_BROWSER = this.executablePath || this.browserChecker.getOrbitaPath;
+
+    const orbitaBrowserExists = await access(ORBITA_BROWSER).then(() => true).catch(() => false);
+    if (!orbitaBrowserExists) {
+      throw new Error(`Orbita browser is not exists on path ${ORBITA_BROWSER}, check executablePath param`);
+    }
 
     await this.createStartup();
     // await this.createBrowserExtension();
@@ -1059,12 +1088,12 @@ class GoLogin {
       return this.stopRemote();
     }
 
-    await this.stopAndCommit({ posting: false }, false);
+    await this.stopAndCommit({ posting: true }, false);
   }
 
   async stopLocal(options) {
     const opts = options || { posting: false };
-    await this.stopAndCommit(options, true);
+    await this.stopAndCommit(opts, true);
   }
 
   async waitDebuggingUrl(delay_ms, try_count=0) {
@@ -1086,7 +1115,7 @@ class GoLogin {
       if (try_count < 3) {
         return this.waitDebuggingUrl(delay_ms, try_count+1);
       }
-      return { 'status': 'failure', wsUrl }
+      return { 'status': 'failure', wsUrl, 'message': 'Check proxy settings', 'profile_id': this.profile_id }
     }
 
     wsUrl = wsUrl.replace('ws://', `wss://`).replace('127.0.0.1', `${this.profile_id}.orbita.gologin.com`)
@@ -1101,7 +1130,7 @@ class GoLogin {
       }
     });
 
-    if(profileResponse.statusCode == 401){
+    if (profileResponse.statusCode === 401){
       throw new Error("invalid token");
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin",
-  "version": "1.0.29",
+  "version": "1.0.33",
   "description": "A high-level API to control Orbita browser over GoLogin API",
   "main": "./gologin.js",
   "repository": {
@@ -26,10 +26,10 @@
     "requestretry": "^4.1.0",
     "rimraf": "^3.0.2",
     "selenium-webdriver": "^4.0.0-alpha.7",
-    "shelljs": "^0.8.4",
     "simple-proxy-agent": "^1.1.0",
     "sqlite": "^4.0.23",
-    "sqlite3": "^5.0.2"
+    "sqlite3": "^5.0.2",
+    "zip-dir": "^2.0.0"
   },
   "bugs": {
     "url": "https://github.com/gologinapp/gologin/issues"

package/selenium/gologin.py

@@ -45,7 +45,7 @@ class GoLogin(object):
         proxy = self.proxy
         proxy_host = ''
         if proxy:
-            if proxy.get('mode')==None:
+            if proxy.get('mode')==None or proxy.get('mode')=='geolocation':
                 proxy['mode'] = 'http'
             proxy_host = proxy.get('host')            
             proxy = self.formatProxyUrl(proxy)