gologin 1.0.31 → 1.0.32

package/gologin.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 const os = require('os');
 const util = require('util');
 const rimraf = util.promisify(require('rimraf'));
+const { access, unlink, writeFile, readFile } = require('fs').promises;
 const exec = util.promisify(require('child_process').exec);
 const { spawn, execFile } = require('child_process');
 const FormData = require('form-data');
@@ -12,7 +13,7 @@ const ProxyAgent = require('simple-proxy-agent');
 const decompress = require('decompress');
 const decompressUnzip = require('decompress-unzip');
 const path = require('path');
-const shell = require('shelljs');
+const zipdir = require('zip-dir');
 
 const BrowserChecker = require('./browser-checker');
 const { BrowserUserDataManager } = require('./browser-user-data-manager');
@@ -54,7 +55,7 @@ class GoLogin {
       this.tmpdir = options.tmpdir;
       if (!fs.existsSync(this.tmpdir)) {
         debug('making tmpdir', this.tmpdir);
-        shell.mkdir('-p', this.tmpdir);
+        fs.mkdirSync(this.tmpdir, { recursive: true })
       }
     }
 
@@ -127,7 +128,7 @@ class GoLogin {
   		throw new Error(`Gologin /browser/${id} response error ${profileResponse.statusCode} INVALID TOKEN OR PROFILE NOT FOUND`);
   	}
 
-    if(profileResponse.statusCode == 401){
+    if (profileResponse.statusCode === 401) {
       throw new Error("invalid token");
     }    
 
@@ -135,12 +136,13 @@ class GoLogin {
   }
 
   async emptyProfile() {
-  	return fs.readFileSync(path.resolve(__dirname, 'gologin_zeroprofile.b64')).toString();
+    return readFile(path.resolve(__dirname, 'gologin_zeroprofile.b64')).then(res => res.toString());
   }
 
   async getProfileS3(s3path) {
     const token = this.access_token;
     debug('getProfileS3 token=', token, 'profile=', this.profile_id, 's3path=', s3path);
+
     if (s3path) { //загрузка профиля из публичного бакета s3 быстрее
       const s3url = `https://gprofiles.gologin.com/${s3path}`.replace(/\s+/mg, '+');
       debug('loading profile from public s3 bucket, url=', s3url);
@@ -151,6 +153,7 @@ class GoLogin {
         debug(`Gologin S3 BUCKET ${s3url} response error ${profileResponse.statusCode}  - use empty`);
         return '';
       }
+
       return Buffer.from(profileResponse.body);
     }
 
@@ -161,15 +164,33 @@ class GoLogin {
       },
       encoding: null
     });
+
     if (profileResponse.statusCode !== 200) {
       debug(`Gologin /browser/${this.profile_id} response error ${profileResponse.statusCode}  - use empty`);
       return '';
     }
+
     return Buffer.from(profileResponse.body);
   }
 
   async postFile(fileName, fileBody) {
     debug('POSTING FILE', fileBody.length);
+    debug('Getting signed URL for S3');
+    const apiUrl = `${API_URL}/browser/${this.profile_id}/storage-signature`;
+
+    const signedUrl = await requests.get(apiUrl, {
+      headers: {
+        Authorization: `Bearer ${this.access_token}`,
+        'user-agent': 'gologin-api',
+      },
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 10 * 1000,
+      fullResponse: false,
+    });
+
+    const [uploadedProfileUrl] = signedUrl.split('?');
+
     const fd = new FormData();
     const boundary = fd.getBoundary();
     const body = Buffer.concat([
@@ -186,26 +207,44 @@ class GoLogin {
       Buffer.from('--'),
       Buffer.from('\r\n')
     ]);
-    await new Promise((resolve) => {
-      let options = {
-        method: 'PATCH',
-        headers: {
-          'Authorization': `Bearer ${this.access_token}`,
-          'Content-Type': 'multipart/form-data; boundary=' + boundary,
-          'Content-Length': body.length
-        },
-        body: body,
-        url: `${API_URL}/browser/${this.profile_id}/profile-s3`,
-      };
-      requests(_.merge(options, {}), () => {
-        resolve();
-      });
+
+    console.log('Uploading profile by signed URL to S3');
+    const bodyBufferBiteLength = Buffer.byteLength(body);
+    console.log('BUFFER SIZE', bodyBufferBiteLength);
+
+    await requests.put(signedUrl, {
+      headers: {
+        'Content-Type': 'application/zip',
+        'Content-Length': bodyBufferBiteLength,
+      },
+      body,
+      maxBodyLength: Infinity,
+      maxContentLength: Infinity,
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 30 * 1000,
+      fullResponse: false,
+    });
+
+    const uploadedProfileMetadata = await requests.head(uploadedProfileUrl, {
+      maxAttempts: 3,
+      retryDelay: 2000,
+      timeout: 10 * 1000,
+      fullResponse: true,
     });
+
+    const uploadedFileLength = +uploadedProfileMetadata.headers['content-length'];
+    if (uploadedFileLength !== bodyBufferBiteLength) {
+      console.log('Uploaded file is incorrect. Retry with China File size:', uploadedFileLength);
+      throw new Error('Uploaded file is incorrect. Retry with China File size: ' + uploadedFileLength);
+    }
+
+    console.log('Profile has been uploaded to S3 successfully');
   }
 
   async emptyProfileFolder() {
     debug('get emptyProfileFolder');
-    const profile = fs.readFileSync(path.resolve(__dirname, 'gologin_zeroprofile.zip'));
+    const profile = await readFile(path.resolve(__dirname, 'gologin_zeroprofile.zip'));
     debug('emptyProfileFolder LENGTH ::', profile.length);
     return profile;
   }
@@ -246,9 +285,8 @@ class GoLogin {
       .then(() => {
         debug('extraction done');
         debug('create uid.json');
-        fs.writeFileSync(path.join(extPath, 'uid.json'), JSON.stringify({uid: that.profile_id}, null, 2))
-        debug('uid.json created', fs.readFileSync(path.join(extPath, 'uid.json')).toString())
-        return extPath;
+        return writeFile(path.join(extPath, 'uid.json'), JSON.stringify({ uid: that.profile_id }, null, 2))
+          .then(() => extPath);
     })
       .catch(async (e) => {
         debug('orbita extension error', e);
@@ -299,7 +337,8 @@ class GoLogin {
       height: parseInt(screenHeight, 10),
     };
 
-    if (!(local && fs.existsSync(this.profile_zip_path))) {
+    const profileZipExists = await access(this.profile_zip_path).then(() => true).catch(() => false);
+    if (!(local && profileZipExists)) {
       try {
         profile_folder = await this.getProfileS3(_.get(profile, 's3Path', ''));
       }
@@ -310,8 +349,8 @@ class GoLogin {
       if (!profile_folder.length) {
         profile_folder = await this.emptyProfileFolder();
       }
-        
-      fs.writeFileSync(this.profile_zip_path, profile_folder);
+
+      await writeFile(this.profile_zip_path, profile_folder);
 
       debug('PROFILE LENGTH', profile_folder.length);
     } else {
@@ -323,20 +362,23 @@ class GoLogin {
     await this.extractProfile(profilePath, this.profile_zip_path);
     debug('extraction done');
 
-    if (fs.existsSync(path.join(profilePath, 'SingletonLock'))) {
+    const singletonLockPath = path.join(profilePath, 'SingletonLock');
+    const singletonLockExists = await access(singletonLockPath).then(() => true).catch(() => false);
+    if (singletonLockExists) {
       debug('removing SingletonLock');
-      fs.unlinkSync(path.join(profilePath, 'SingletonLock'));
+      await unlink(singletonLockPath);
       debug('SingletonLock removed');
     }
 
     const pref_file_name = path.join(profilePath, 'Default', 'Preferences');
     debug('reading', pref_file_name);
 
-    if (!fs.existsSync(pref_file_name)) {
+    const prefFileExists = await access(pref_file_name).then(() => true).catch(() => false);
+    if (!prefFileExists) {
       debug('Preferences file not exists waiting', pref_file_name);
     }
 
-    const preferences_raw = fs.readFileSync(pref_file_name);
+    const preferences_raw = await readFile(pref_file_name);
     let preferences = JSON.parse(preferences_raw.toString());
     let proxy = _.get(profile, 'proxy');
     let name = _.get(profile, 'name');
@@ -436,7 +478,7 @@ class GoLogin {
       await BrowserUserDataManager.composeFonts(families, profilePath, this.differentOs);
     }
 
-    fs.writeFileSync(path.join(profilePath, 'Default', 'Preferences'), JSON.stringify(_.merge(preferences, {
+    await writeFile(path.join(profilePath, 'Default', 'Preferences'), JSON.stringify(_.merge(preferences, {
       gologin
     })));
 
@@ -450,12 +492,14 @@ class GoLogin {
 
   async commitProfile() {
     const data = await this.getProfileDataToUpdate();
+
     debug('begin updating', data.length);
     if (!data.length) {
       debug('WARN: profile zip data empty - SKIPPING PROFILE COMMIT');
 
       return;
     }
+
     try {
       debug('Patching profile');
       await this.postFile('profile', data);
@@ -463,6 +507,7 @@ class GoLogin {
     catch (e) {
       debug('CANNOT COMMIT PROFILE', e);
     }
+
     debug('COMMIT COMPLETED');
   }
 
@@ -482,16 +527,16 @@ class GoLogin {
 
   async checkPortAvailable(port) {
     debug('CHECKING PORT AVAILABLE', port);
+
     try {
       const { stdout, stderr } = await exec(`lsof -i:${port}`);
-      if (
-        stdout && stdout.match(/LISTEN/gmi)
-      ) {
+      if (stdout && stdout.match(/LISTEN/gmi)) {
         debug(`PORT ${port} IS BUSY`)
         return false;
       }
-    } catch (e) { }
+    } catch (e) {}
     debug(`PORT ${port} IS OPEN`);
+
     return true;
   }
 
@@ -507,7 +552,7 @@ class GoLogin {
 
   async getTimeZone(proxy) {
     debug('getting timeZone proxy=', proxy);
-    if(this.timezone){
+    if (this.timezone) {
       debug('getTimeZone from options', this.timezone);
       this._tz = this.timezone;
       return this._tz.timezone;
@@ -604,7 +649,7 @@ class GoLogin {
 
   async spawnBrowser() {
     let remote_debugging_port = this.remote_debugging_port;
-    if(!remote_debugging_port){
+    if (!remote_debugging_port) {
       remote_debugging_port = await this.getRandomPort();
     } 
     
@@ -706,7 +751,9 @@ class GoLogin {
     if (this.is_stopping) {
       return true;
     }
-    const is_posting = options.postings || false;
+    const is_posting = options.posting ||
+      options.postings || // backward compability
+      false;
 
     if (this.uploadCookiesToServer) {
       await this.uploadProfileCookiesToServer();
@@ -782,34 +829,39 @@ class GoLogin {
 
   async getProfileDataToUpdate() {
     const zipPath = path.join(this.tmpdir, `gologin_${this.profile_id}_upload.zip`);
-    try {
-      fs.unlinkSync(zipPath);
-    }
-    catch (e) {
+    const zipExists = await access(zipPath).then(() => true).catch(() => false);
+    if (zipExists) {
+      await unlink(zipPath);
     }
+
     await this.sanitizeProfile();
     debug('profile sanitized');
-    await new Promise(resolve => {
-      debug('begin zipping');
-      execFile(`cd ${this.profilePath()}; /usr/bin/zip`, [
-        `-r ${zipPath}`,
-        '*'
-      ], {
-        shell: true
-      }, () => {
-        debug('zipping done');
-        resolve();
-      });
-    });
-    debug('PROFILE ZIP CREATED', this.profilePath(), zipPath);
+
+    const profilePath = this.profilePath();
+    await new Promise((resolve, reject) => zipdir(profilePath,
+      {
+        saveTo: zipPath,
+        filter: (path) => !/RunningChromeVersion/.test(path),
+      }, (err, buffer) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        resolve(buffer);
+      })
+    )
+
+    debug('PROFILE ZIP CREATED', profilePath, zipPath);
+
+    let data = '';
     try {
-      const data = fs.readFileSync(zipPath);
-      return data;
-    }
-    catch (e) {
+      data = await readFile(zipPath);
+    } catch (e) {
       debug('saveprofile error', e);
-      return '';
     }
+
+    return data;
   }
 
   async profileExists() {
@@ -853,11 +905,11 @@ class GoLogin {
     const fingerprint = await this.getRandomFingerprint(options);
     debug("fingerprint=", fingerprint)
     
-    if(fingerprint.statusCode == 500){
+    if (fingerprint.statusCode === 500) {
       throw new Error("no valid random fingerprint check os param");
     }
 
-    if(fingerprint.statusCode == 401){
+    if (fingerprint.statusCode === 401) {
       throw new Error("invalid token");
     }
 
@@ -887,7 +939,7 @@ class GoLogin {
     let user_agent = options.navigator?.userAgent;
     let orig_user_agent = json.navigator.userAgent;
     Object.keys(options).map((e)=>{ json[e] = options[e] });
-    if(user_agent=='random'){
+    if (user_agent === 'random') {
       json.navigator.userAgent = orig_user_agent;
     }
     // console.log('profileOptions', json);
@@ -900,11 +952,11 @@ class GoLogin {
       json,
     });
 
-    if(response.body.statusCode==400){
+    if (response.body.statusCode === 400) {
       throw new Error(`gologin failed account creation with status code, ${data.statusCode} DATA  ${JSON.stringify(response.body.message)}`);
     }
 
-    if(response.body.statusCode==500){
+    if (response.body.statusCode === 500) {
       throw new Error(`gologin failed account creation with status code, ${data.statusCode}`);
     }
     debug(JSON.stringify(response.body));
@@ -1045,7 +1097,8 @@ class GoLogin {
     
     const ORBITA_BROWSER = this.executablePath || this.browserChecker.getOrbitaPath;
 
-    if(!fs.existsSync(ORBITA_BROWSER)){
+    const orbitaBrowserExists = await access(ORBITA_BROWSER).then(() => true).catch(() => false);
+    if (!orbitaBrowserExists) {
       throw new Error(`Orbita browser is not exists on path ${ORBITA_BROWSER}, check executablePath param`);
     }
 
@@ -1071,12 +1124,12 @@ class GoLogin {
       return this.stopRemote();
     }
 
-    await this.stopAndCommit({ posting: false }, false);
+    await this.stopAndCommit({ posting: true }, false);
   }
 
   async stopLocal(options) {
     const opts = options || { posting: false };
-    await this.stopAndCommit(options, true);
+    await this.stopAndCommit(opts, true);
   }
 
   async waitDebuggingUrl(delay_ms, try_count=0) {
@@ -1113,7 +1166,7 @@ class GoLogin {
       }
     });
 
-    if(profileResponse.statusCode == 401){
+    if (profileResponse.statusCode === 401){
       throw new Error("invalid token");
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin",
-  "version": "1.0.31",
+  "version": "1.0.32",
   "description": "A high-level API to control Orbita browser over GoLogin API",
   "main": "./gologin.js",
   "repository": {
@@ -26,10 +26,10 @@
     "requestretry": "^4.1.0",
     "rimraf": "^3.0.2",
     "selenium-webdriver": "^4.0.0-alpha.7",
-    "shelljs": "^0.8.4",
     "simple-proxy-agent": "^1.1.0",
     "sqlite": "^4.0.23",
-    "sqlite3": "^5.0.2"
+    "sqlite3": "^5.0.2",
+    "zip-dir": "^2.0.0"
   },
   "bugs": {
     "url": "https://github.com/gologinapp/gologin/issues"