gologin 1.0.40 → 1.0.43

package/browser-user-data-manager.js

@@ -113,6 +113,93 @@ class BrowserUserDataManager {
     const result = fileContent.replace(/\$\$GOLOGIN_FONTS\$\$/g, path.join(profilePath, FONTS_DIR_NAME));
     await writeFile(path.join(profilePath, 'Default', 'fonts_config'), result);
   }
+
+  static setExtPaths(settings = {}, profileExtensionsCheckRes = []) {
+    const formattedLocalExtArray = profileExtensionsCheckRes.map((el) => {
+      const [extFolderName = ''] = el.split(path.sep).reverse();
+      const [originalId] = extFolderName.split('@');
+      if (!originalId) {
+        return null;
+      }
+
+      return {
+        path: el,
+        originalId,
+      }
+    }).filter(Boolean);
+
+    if (!formattedLocalExtArray.length) {
+      return;
+    }
+
+    const extensionsSettings = settings.extensions?.settings || {};
+    const extensionsEntries = Object.entries(extensionsSettings);
+
+    extensionsEntries.forEach((extensionObj) => {
+      const [extensionsId] = extensionObj;
+      const localExtObj = formattedLocalExtArray.find(el => el.originalId === extensionsId);
+      if (!localExtObj) {
+        return;
+      }
+
+      extensionsSettings[extensionsId].path = localExtObj?.path || '';
+    });
+
+    return extensionsSettings;
+  }
+
+  static async setOriginalExtPaths(settings = {}, originalExtensionsFolder = '') {
+    if (!originalExtensionsFolder) {
+      return null;
+    }
+
+    const extensionsSettings = settings.extensions?.settings || {};
+    const extensionsEntries = Object.entries(extensionsSettings);
+
+    const originalExtensionsList = await readdir(originalExtensionsFolder).catch(() => []);
+    if (!originalExtensionsList.length) {
+      return null;
+    }
+
+    const promises = originalExtensionsList.map(async (originalId) => {
+      const extFolderPath = path.join(originalExtensionsFolder, originalId);
+      const extFolderContent = await readdir(extFolderPath);
+      if (!extFolderPath.length) {
+        return {};
+      }
+
+      if (extFolderContent.includes('manifest.json')) {
+        return {
+          originalId,
+          path: path.join(originalExtensionsFolder, originalId),
+        };
+      }
+
+      const [version] = extFolderContent;
+      return {
+        originalId,
+        path: path.join(originalExtensionsFolder, originalId, version),
+      };
+    });
+    const originalExtPaths = await Promise.all(promises);
+
+    extensionsEntries.forEach((extensionObj) => {
+      const [extensionsId] = extensionObj;
+      const extPath = extensionsSettings[extensionsId].path;
+      if (!/chrome-extensions/.test(extPath)) {
+        return;
+      }
+
+      const originalExtPath = originalExtPaths.find(el => el.originalId === extensionsId);
+      if (!originalExtPath) {
+        return;
+      }
+
+      extensionsSettings[extensionsId].path = originalExtPath.path || '';
+    });
+
+    return extensionsSettings;
+  }
 }
 
 module.exports = {

package/extensions-extractor.js

@@ -0,0 +1,55 @@
+const { access, unlink } = require('fs').promises;
+const decompress = require('decompress');
+const decompressUnzip = require('decompress-unzip');
+
+class ExtensionsExtractor {
+  static extractExtension(source, dest) {
+    if (!(source && dest)) {
+      throw new Error('Missing parameter');
+    }
+
+    return access(source)
+      .then(() =>
+        withRetry({
+          fn() {
+            return decompress(source, dest, {
+              plugins: [decompressUnzip()],
+              filter: file => !file.path.endsWith('/')
+            })
+          }
+        })
+      );
+  }
+
+  static deleteExtensionArchive(dest) {
+    if (!dest) {
+      throw new Error('Missing parameter');
+    }
+
+    return access(dest)
+      .then(
+        () => unlink(dest),
+        () => Promise.resolve()
+      )
+  }
+}
+
+const withRetry = optionsOrUndefined => {
+  const opts = optionsOrUndefined || {};
+  const callCounter = opts.callCounter || 1;
+  const fnToProducePromise = opts.fn;
+  const callLimit = opts.limit || 5;
+  delete opts.callCounter;
+  return fnToProducePromise(opts).catch(err => {
+    console.error(err);
+    if (callCounter >= callLimit) {
+      return Promise.reject(err);
+    }
+    opts.callCounter = callCounter + 1;
+    return new Promise(resolve => process.nextTick(resolve)).then(() =>
+      withRetry(opts)
+    );
+  });
+};
+
+module.exports = ExtensionsExtractor;

package/extensions-manager.js

@@ -0,0 +1,322 @@
+const path = require('path');
+const request = require('requestretry').defaults({ timeout: 60000 });
+const fs = require('fs');
+const { mkdir, readdir, rmdir } = require('fs').promises;
+const os = require('os');
+
+const  ExtensionsExtractor = require('./extensions-extractor');
+
+const HOMEDIR = os.homedir();
+const CHROME_EXT_DIR_NAME = 'chrome-extensions';
+const EXTENSIONS_PATH = path.join(HOMEDIR, '.gologin', 'extensions');
+const CHROME_EXTENSIONS_PATH = path.join(EXTENSIONS_PATH, CHROME_EXT_DIR_NAME);
+const EXTENSION_URL = 'https://clients2.google.com/service/update2/crx?response=redirect&acceptformat=crx2,crx3&x=id%3D{ext_id}%26uc&prodversion=97.0.4692.71';
+
+class ExtensionsManager {
+  #USER_AGENT = '';
+  #API_BASE_URL = '';
+  #ACCESS_TOKEN = '';
+  #existedChromeExtensions = [];
+  #inited = false;
+  #useLocalExtStorage = false;
+  #deleteProfileExtFolders = false;
+  #deleteWidevineCdmFolder = false;
+
+  constructor() {
+    if (!ExtensionsManager.instance) {
+      ExtensionsManager.instance = this;
+    }
+
+    return ExtensionsManager.instance;
+  }
+
+  get isInited() { return this.#inited }
+  get useLocalExtStorage() { return this.#useLocalExtStorage }
+  get deleteProfileExtFolders() { return this.#deleteProfileExtFolders }
+  get deleteWidevineCdmFolder() { return this.#deleteWidevineCdmFolder }
+
+  set userAgent(userAgent) {
+    if (!userAgent) {
+      return;
+    }
+
+    this.#USER_AGENT = userAgent;
+  }
+
+  set accessToken(accessToken) {
+    if (!accessToken) {
+      return;
+    }
+
+    this.#ACCESS_TOKEN = accessToken;
+  }
+
+  set apiUrl(apiUrl) {
+    if (!apiUrl) {
+      return;
+    }
+
+    this.#API_BASE_URL = apiUrl;
+  }
+
+  init() {
+    if (this.#inited) {
+      return Promise.resolve();
+    }
+
+    return mkdir(CHROME_EXTENSIONS_PATH, { recursive: true })
+      .then(() => readdir(CHROME_EXTENSIONS_PATH))
+      .then(filesList => {
+        this.#existedChromeExtensions = filesList;
+        this.#inited = true;
+      })
+      .catch((e) => console.log('ExtensionsManager init error:', e));
+  }
+
+  get existedChromeExtensionsList() {
+    return this.#existedChromeExtensions;
+  }
+
+  async checkChromeExtensions(profileExtensions = []) {
+    if (!(Array.isArray(profileExtensions) && profileExtensions.length)) {
+      return [];
+    }
+
+    const extensionsToDownload = this.#getExtensionsToDownload(profileExtensions);
+    if (!extensionsToDownload) {
+      return [];
+    }
+
+    const downloadedArchives = await this.downloadChromeExtensions(extensionsToDownload);
+    const filteredArchives = downloadedArchives.filter(Boolean);
+    const promises = composeExtractionPromises(filteredArchives);
+
+    await Promise.all(promises);
+    return this.getExtensionsStrToIncludeAsOrbitaParam(profileExtensions);
+  }
+
+  #getExtensionsToDownload(profileExtensions) {
+    const existedOriginalIds = this.#existedChromeExtensions.map((val) => {
+      const [originalId] = val.split('@');
+      return originalId;
+    });
+
+    return profileExtensions.reduce((res, val) => {
+      const [originalId] = val.split('@');
+      const extensionExists = existedOriginalIds.includes(originalId);
+      if (!extensionExists) {
+        res.push(val);
+      }
+
+      return res;
+    }, []);
+  }
+
+  async downloadChromeExtensions(idsToDownload = []) {
+    if (!(Array.isArray(idsToDownload) && idsToDownload.length)) {
+      return [];
+    }
+
+    const promises = idsToDownload.map(async (id) => {
+      const [originalId] = id.split('@');
+      const extUrl = EXTENSION_URL.replace('{ext_id}', originalId);
+
+      const uploadedProfileMetadata = await getExtMetadata(extUrl);
+
+      const reqPath = uploadedProfileMetadata.req.path;
+      const extVer = getExtVersion(reqPath);
+
+      const buffer = await new Promise((res) => {
+        const chunks = []
+        request.get(extUrl, {
+          maxAttempts: 3,
+          retryDelay: 1000,
+          timeout: 8 * 1000,
+          fullResponse: false,
+        })
+          .on('data', (data) => chunks.push(data))
+          .on('end', () => res(Buffer.concat(chunks)));
+      });
+
+      let zipExt;
+      try {
+        zipExt = crxToZip(buffer);
+      } catch (e) {
+        console.log(e);
+        return '';
+      }
+
+      const archiveZipPath = path.join(CHROME_EXTENSIONS_PATH, originalId + '@' + extVer + '.zip');
+
+      const archiveZip = fs.createWriteStream(archiveZipPath);
+      archiveZip.write(zipExt);
+      archiveZip.close();
+
+      return new Promise(r => archiveZip.on('close', () => r(archiveZipPath)));
+    });
+
+    return Promise.all(promises);
+  }
+
+  async getExtensionsPolicies() {
+    const globalExtConfig = await request.get(`${this.#API_BASE_URL}/gologin-settings/chrome_ext_policies`, {
+      headers: {
+        Authorization: `Bearer ${this.#ACCESS_TOKEN}`,
+        'user-agent': this.#USER_AGENT,
+      },
+      json: true,
+      maxAttempts: 2,
+      retryDelay: 1000,
+      timeout: 10 * 1000,
+      fullResponse: false,
+    });
+
+    const chromeExtPolicies = globalExtConfig?.chromeExtPolicies || {};
+    const {
+      useLocalExtStorage = false,
+      deleteProfileExtFolders = false,
+      deleteWidevineCdmFolder = false,
+    } = chromeExtPolicies;
+
+    this.#useLocalExtStorage = useLocalExtStorage;
+    this.#deleteProfileExtFolders = deleteProfileExtFolders;
+    this.#deleteWidevineCdmFolder = deleteWidevineCdmFolder;
+  }
+
+  async getExtensionsStrToIncludeAsOrbitaParam(profileExtensions = []) {
+    if (!(Array.isArray(profileExtensions) && profileExtensions.length)) {
+      return [];
+    }
+
+    const chromeExtList = await readdir(CHROME_EXTENSIONS_PATH);
+    if (!chromeExtList) {
+      return [];
+    }
+
+    const formattedIdsList = chromeExtList.map((el) => {
+      const [originalId] = el.split('@');
+      return {
+        originalId,
+        folderName: el,
+      };
+    });
+
+    return profileExtensions.map((el) => {
+      const extExisted = formattedIdsList.find(chromeExtPathElem => chromeExtPathElem.originalId === el);
+      if (!extExisted) {
+        return '';
+      }
+
+      return path.join(CHROME_EXTENSIONS_PATH, extExisted.folderName);
+    }).filter(Boolean);
+  }
+
+  async updateExtensions() {
+    const fileList = await readdir(CHROME_EXTENSIONS_PATH).catch(() => []);
+    if (!fileList.length) {
+      return;
+    }
+
+    const oldFolders = [];
+
+    const versionCheckPromises = fileList.map(async (extension) => {
+      if (!extension.includes('@')) {
+        return '';
+      }
+
+      const [originalId, currentVersion] = extension.split('@');
+      const extUrl = EXTENSION_URL.replace('{ext_id}', originalId);
+      const uploadedProfileMetadata = await getExtMetadata(extUrl);
+      const reqPath = uploadedProfileMetadata.req.path;
+      const availableVersion = getExtVersion(reqPath);
+
+      if (currentVersion === availableVersion) {
+        return '';
+      }
+
+      oldFolders.push(path.join(CHROME_EXTENSIONS_PATH, extension));
+      return originalId;
+    });
+
+    const extensionsNames = (await Promise.all(versionCheckPromises)).filter(Boolean);
+    const archivesPaths = (await this.downloadChromeExtensions(extensionsNames)).filter(Boolean);
+    const extractionPromises = composeExtractionPromises(archivesPaths);
+    await Promise.all(extractionPromises);
+
+    const removeFoldersPromises = oldFolders.map(folder => (
+      rmdir(folder, { recursive: true, maxRetries: 3 }).catch(() => {})
+    ));
+
+    return Promise.all(removeFoldersPromises);
+  }
+}
+
+const crxToZip = (buf) => {
+  if (buf[0] === 80 && buf[1] === 75 && buf[2] === 3 && buf[3] === 4) {
+    return buf;
+  }
+
+  if (!(buf[0] === 67 || buf[1] === 114 || buf[2] === 50 || buf[3] === 52)) {
+    throw new Error('Invalid header: Does not start with Cr24');
+  }
+
+  const isV3 = buf[4] === 3;
+  const isV2 = buf[4] === 2;
+
+  if (!(isV2 || isV3) || buf[5] || buf[6] || buf[7]) {
+    throw new Error('Unexpected crx format version number.');
+  }
+
+  if (isV2) {
+    const publicKeyLength = calcLength(buf[8], buf[9], buf[10], buf[11]);
+    const signatureLength = calcLength(buf[12], buf[13], buf[14], buf[15]);
+
+    const zipStartOffset = 16 + publicKeyLength + signatureLength;
+    return buf.slice(zipStartOffset, buf.length);
+  }
+
+  const headerSize = calcLength(buf[8], buf[9], buf[10], buf[11]);
+  const zipStartOffset = 12 + headerSize;
+
+  return buf.slice(zipStartOffset, buf.length);
+}
+
+const calcLength = (a, b, c, d) => {
+  let length = 0;
+
+  length += a << 0;
+  length += b << 8;
+  length += c << 16;
+  length += d << 24 >>> 0;
+  return length;
+}
+
+const getExtMetadata = (extUrl) => (
+  request.head(extUrl, {
+    maxAttempts: 3,
+    retryDelay: 2000,
+    timeout: 2 * 1000,
+    fullResponse: true,
+  })
+);
+
+const getExtVersion = (metadata) => {
+  const [extFullName = ''] = metadata.split('/').reverse();
+  const [extName = ''] = extFullName.split('.');
+  const splitExtName = extName.split('_');
+  splitExtName.shift();
+  return splitExtName.join('_');
+};
+
+const composeExtractionPromises = (filteredArchives) => (
+  filteredArchives.map((extArchivePath) => {
+    const [archiveName = ''] = extArchivePath.split(path.sep).reverse();
+    const [destFolder] = archiveName.split('.');
+    return ExtensionsExtractor.extractExtension(extArchivePath, path.join(CHROME_EXTENSIONS_PATH, destFolder))
+      .then(() => ExtensionsExtractor.deleteExtensionArchive(extArchivePath))
+  })
+);
+
+module.exports = ExtensionsManager;
+
+

package/gologin.js

@@ -13,12 +13,13 @@ const decompress = require('decompress');
 const decompressUnzip = require('decompress-unzip');
 const path = require('path');
 const zipdir = require('zip-dir');
+const https = require('https');
 
 const BrowserChecker = require('./browser-checker');
 const { BrowserUserDataManager } = require('./browser-user-data-manager');
 const { CookiesManager } = require('./cookies-manager');
 const fontsCollection = require('./fonts');
-const https = require('https');
+const ExtensionsManager = require('./extensions-manager');
 
 const SEPARATOR = path.sep;
 const API_URL = 'https://api.gologin.com';
@@ -360,13 +361,50 @@ class GoLogin {
 
     const prefFileExists = await access(pref_file_name).then(() => true).catch(() => false);
     if (!prefFileExists) {
-      debug('Preferences file not exists waiting', pref_file_name);
+      debug('Preferences file not exists waiting', pref_file_name, '. Using empty profile');
+      profile_folder = await this.emptyProfileFolder();
+      await writeFile(this.profile_zip_path, profile_folder);      
+      await this.extractProfile(profilePath, this.profile_zip_path);
     }
 
     const preferences_raw = await readFile(pref_file_name);
     let preferences = JSON.parse(preferences_raw.toString());    
     let proxy = _.get(profile, 'proxy');
     let name = _.get(profile, 'name');
+    const chromeExtensions = _.get(profile, 'chromeExtensions');
+
+    if (chromeExtensions.length) {
+      const ExtensionsManagerInst = new ExtensionsManager();
+      ExtensionsManagerInst.apiUrl = API_URL;
+      await ExtensionsManagerInst.init()
+        .then(() => ExtensionsManagerInst.updateExtensions())
+        .catch(() => {});
+      ExtensionsManagerInst.accessToken = this.access_token;
+
+      await ExtensionsManagerInst.getExtensionsPolicies();
+      let profileExtensionsCheckRes = [];
+
+      if (ExtensionsManagerInst.useLocalExtStorage) {
+        profileExtensionsCheckRes = await ExtensionsManagerInst.checkChromeExtensions(chromeExtensions).catch((e) => {
+          console.log('checkChromeExtensions error: ', e);
+          return [];
+        });
+      }
+
+      let extSettings;
+      if (ExtensionsManagerInst.useLocalExtStorage && profileExtensionsCheckRes.length) {
+        extSettings = BrowserUserDataManager.setExtPaths(preferences, profileExtensionsCheckRes);
+      } else if (!ExtensionsManagerInst.useLocalExtStorage) {
+        const originalExtensionsFolder = path.join(profilePath, 'Default', 'Extensions');
+        extSettings = await BrowserUserDataManager.setOriginalExtPaths(preferences, originalExtensionsFolder);
+      }
+
+      if (extSettings) {
+        const currentExtSettings = preferences.extensions || {};
+        currentExtSettings.settings = extSettings
+        preferences.extensions = currentExtSettings;
+      }
+    }
 
     if (proxy.mode === 'gologin' || proxy.mode === 'tor') {
       const autoProxyServer = _.get(profile, 'autoProxyServer');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gologin",
-  "version": "1.0.40",
+  "version": "1.0.43",
   "description": "A high-level API to control Orbita browser over GoLogin API",
   "main": "./gologin.js",
   "repository": {