gologin-commonjs 2.1.1

package/dist/src/cookies/cookies-manager.js

@@ -0,0 +1,164 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.unixToLDAP = exports.loadCookiesFromFile = exports.ldapToUnix = exports.getDB = exports.getCookiesFilePath = exports.getChunckedInsertValues = exports.chunk = exports.buildCookieURL = void 0;
+var _sqlite = require("sqlite");
+var _sqlite2 = _interopRequireDefault(require("sqlite3"));
+var _fs = require("fs");
+var _path = require("path");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  access
+} = _fs.promises;
+const {
+  Database,
+  OPEN_READONLY
+} = _sqlite2.default;
+const MAX_SQLITE_VARIABLES = 76;
+const SAME_SITE = {
+  '-1': 'unspecified',
+  0: 'no_restriction',
+  1: 'lax',
+  2: 'strict'
+};
+const getDB = (filePath, readOnly = true) => {
+  const connectionOpts = {
+    filename: filePath,
+    driver: Database
+  };
+  if (readOnly) {
+    connectionOpts.mode = OPEN_READONLY;
+  }
+  return (0, _sqlite.open)(connectionOpts);
+};
+exports.getDB = getDB;
+const getChunckedInsertValues = cookiesArr => {
+  const todayUnix = Math.floor(new Date().getTime() / 1000.0);
+  const chunckedCookiesArr = chunk(cookiesArr, MAX_SQLITE_VARIABLES);
+  return chunckedCookiesArr.map(cookies => {
+    const queryPlaceholders = cookies.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ');
+    const query = `insert or replace into cookies (creation_utc, host_key, top_frame_site_key, name, value, encrypted_value, path, expires_utc, is_secure, is_httponly, last_access_utc, has_expires, is_persistent, priority, samesite, source_scheme, source_port, is_same_party, last_update_utc) values ${queryPlaceholders}`;
+    const queryParams = cookies.flatMap(cookie => {
+      const creationDate = cookie.creationDate ? cookie.creationDate : unixToLDAP(todayUnix);
+      let expirationDate = cookie.session ? 0 : unixToLDAP(cookie.expirationDate);
+      const encryptedValue = cookie.value;
+      const samesite = Object.keys(SAME_SITE).find(key => SAME_SITE[key] === (cookie.sameSite || '-1'));
+      const isSecure = cookie.name.startsWith('__Host-') || cookie.name.startsWith('__Secure-') ? 1 : Number(cookie.secure);
+      const sourceScheme = isSecure === 1 ? 2 : 1;
+      const sourcePort = isSecure === 1 ? 443 : 80;
+      // eslint-disable-next-line no-undefined
+      let isPersistent = [undefined, null].includes(cookie.session) ? Number(expirationDate !== 0) : Number(!cookie.session);
+      if (/^(\.)?mail.google.com$/.test(cookie.domain) && cookie.name === 'COMPASS') {
+        expirationDate = 0;
+        isPersistent = 0;
+      }
+      return [creationDate, cookie.domain, '',
+      // top_frame_site_key
+      cookie.name, '',
+      // value
+      encryptedValue, cookie.path, expirationDate, isSecure, Number(cookie.httpOnly), 0,
+      // last_access_utc
+      expirationDate === 0 ? 0 : 1,
+      // has_expires
+      isPersistent, 1,
+      // default priority value (https://github.com/chromium/chromium/blob/main/net/cookies/cookie_constants.h)
+      samesite, sourceScheme, sourcePort, 0,
+      // is_same_party
+      0 // last_update_utc
+      ];
+    });
+    return [query, queryParams];
+  });
+};
+exports.getChunckedInsertValues = getChunckedInsertValues;
+const loadCookiesFromFile = async filePath => {
+  let db;
+  const cookies = [];
+  try {
+    db = await getDB(filePath);
+    const cookiesRows = await db.all('select * from cookies');
+    for (const row of cookiesRows) {
+      const {
+        host_key,
+        name,
+        encrypted_value,
+        path,
+        is_secure,
+        is_httponly,
+        expires_utc,
+        is_persistent,
+        samesite,
+        creation_utc
+      } = row;
+      cookies.push({
+        url: buildCookieURL(host_key, is_secure, path),
+        domain: host_key,
+        name,
+        value: encrypted_value,
+        path,
+        sameSite: SAME_SITE[samesite],
+        secure: Boolean(is_secure),
+        httpOnly: Boolean(is_httponly),
+        hostOnly: !host_key.startsWith('.'),
+        session: !is_persistent,
+        expirationDate: ldapToUnix(expires_utc),
+        creationDate: ldapToUnix(creation_utc)
+      });
+    }
+  } catch (error) {
+    console.log(error);
+  } finally {
+    db && (await db.close());
+  }
+  return cookies;
+};
+exports.loadCookiesFromFile = loadCookiesFromFile;
+const unixToLDAP = unixtime => {
+  if (unixtime === 0) {
+    return unixtime;
+  }
+  const win32filetime = new Date(Date.UTC(1601, 0, 1)).getTime() / 1000;
+  const sum = unixtime - win32filetime;
+  return sum * 1000000;
+};
+exports.unixToLDAP = unixToLDAP;
+const ldapToUnix = ldap => {
+  const ldapLength = ldap.toString().length;
+  if (ldap === 0 || ldapLength > 18) {
+    return ldap;
+  }
+  let _ldap = ldap;
+  if (ldapLength < 18) {
+    _ldap = Number(_ldap + '0'.repeat(18 - ldapLength));
+  }
+  const win32filetime = new Date(Date.UTC(1601, 0, 1)).getTime();
+  return (_ldap / 10000 + win32filetime) / 1000;
+};
+exports.ldapToUnix = ldapToUnix;
+const buildCookieURL = (domain, secure, path) => {
+  let domainWithoutDot = domain;
+  if (domain.startsWith('.')) {
+    domainWithoutDot = domain.substr(1);
+  }
+  return 'http' + (secure ? 's' : '') + '://' + domainWithoutDot + path;
+};
+exports.buildCookieURL = buildCookieURL;
+const chunk = (arr, chunkSize = 1, cache = []) => {
+  const tmp = [...arr];
+  if (chunkSize <= 0) {
+    return cache;
+  }
+  while (tmp.length) {
+    cache.push(tmp.splice(0, chunkSize));
+  }
+  return cache;
+};
+exports.chunk = chunk;
+const getCookiesFilePath = async (profileId, tmpdir) => {
+  const baseCookiesFilePath = (0, _path.join)(tmpdir, `gologin_profile_${profileId}`, 'Default', 'Cookies');
+  const bypassCookiesFilePath = (0, _path.join)(tmpdir, `gologin_profile_${profileId}`, 'Default', 'Network', 'Cookies');
+  return access(baseCookiesFilePath).then(() => baseCookiesFilePath).catch(() => access(bypassCookiesFilePath).then(() => bypassCookiesFilePath).catch(() => baseCookiesFilePath));
+};
+exports.getCookiesFilePath = getCookiesFilePath;

package/dist/src/extensions/extensions-extractor.js

@@ -0,0 +1,50 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.extractExtension = exports.deleteExtensionArchive = void 0;
+var _decompress = _interopRequireDefault(require("decompress"));
+var _decompressUnzip = _interopRequireDefault(require("decompress-unzip"));
+var _fs = require("fs");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  access,
+  unlink
+} = _fs.promises;
+const extractExtension = (source, dest) => {
+  if (!(source && dest)) {
+    throw new Error('Missing parameter');
+  }
+  return access(source).then(() => withRetry({
+    fn() {
+      return (0, _decompress.default)(source, dest, {
+        plugins: [(0, _decompressUnzip.default)()],
+        filter: file => !file.path.endsWith('/')
+      });
+    }
+  }));
+};
+exports.extractExtension = extractExtension;
+const deleteExtensionArchive = dest => {
+  if (!dest) {
+    throw new Error('Missing parameter');
+  }
+  return access(dest).then(() => unlink(dest), () => Promise.resolve());
+};
+exports.deleteExtensionArchive = deleteExtensionArchive;
+const withRetry = optionsOrUndefined => {
+  const opts = optionsOrUndefined || {};
+  const callCounter = opts.callCounter || 1;
+  const fnToProducePromise = opts.fn;
+  const callLimit = opts.limit || 5;
+  delete opts.callCounter;
+  return fnToProducePromise(opts).catch(err => {
+    console.error(err);
+    if (callCounter >= callLimit) {
+      return Promise.reject(err);
+    }
+    opts.callCounter = callCounter + 1;
+    return new Promise(resolve => process.nextTick(resolve)).then(() => withRetry(opts));
+  });
+};

package/dist/src/extensions/extensions-manager.js

@@ -0,0 +1,301 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = exports.ExtensionsManager = void 0;
+var _fs = require("fs");
+var _path = require("path");
+var _requestretry = _interopRequireDefault(require("requestretry"));
+var _common = require("../utils/common.js");
+var _userExtensionsManager = _interopRequireDefault(require("./user-extensions-manager.js"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const {
+  mkdir,
+  readdir,
+  rmdir,
+  unlink
+} = _fs.promises;
+const EXTENSION_URL = 'https://clients2.google.com/service/update2/crx?response=redirect&acceptformat=crx2,crx3&x=id%3D{ext_id}%26uc&prodversion=97.0.4692.71';
+class ExtensionsManager extends _userExtensionsManager.default {
+  #existedChromeExtensions = [];
+  #inited = false;
+  #useLocalExtStorage = false;
+  #useCookiesExt = false;
+  #deleteProfileExtFolders = false;
+  #extensionsUpdating = true;
+  constructor() {
+    super();
+    if (!ExtensionsManager.instance) {
+      ExtensionsManager.instance = this;
+    }
+    return ExtensionsManager.instance;
+  }
+  async init() {
+    if (this.#inited) {
+      return Promise.resolve();
+    }
+    const promises = [mkdir(_common.CHROME_EXTENSIONS_PATH, {
+      recursive: true
+    }).then(() => readdir(_common.CHROME_EXTENSIONS_PATH)).then(filesList => {
+      this.#existedChromeExtensions = filesList.filter(extPath => !extPath.includes('.zip'));
+      return filesList.map(fileName => fileName.includes('.zip') ? unlink((0, _path.join)(_common.CHROME_EXTENSIONS_PATH, fileName)) : Promise.resolve());
+    }).then(promisesToDelete => Promise.all(promisesToDelete)).catch(e => console.log('ExtensionsManager init error:', e)), mkdir(_common.USER_EXTENSIONS_PATH, {
+      recursive: true
+    }).then(() => readdir(_common.USER_EXTENSIONS_PATH)).then(filesList => {
+      this.existedUserExtensions = filesList.filter(extPath => !extPath.includes('.zip'));
+      return filesList.map(fileName => fileName.includes('.zip') ? unlink((0, _path.join)(_common.USER_EXTENSIONS_PATH, fileName)) : Promise.resolve());
+    }).then(promisesToDelete => Promise.all(promisesToDelete)).catch(e => console.log('error creating user extensions folder:', e))];
+    return Promise.all(promises).then(() => this.#inited = true);
+  }
+  get isInited() {
+    return this.#inited;
+  }
+  get useLocalExtStorage() {
+    return this.#useLocalExtStorage;
+  }
+  get deleteProfileExtFolders() {
+    return this.#deleteProfileExtFolders;
+  }
+  get useCookiesExt() {
+    return this.#useCookiesExt;
+  }
+  get existedChromeExtensionsList() {
+    return this.#existedChromeExtensions;
+  }
+  async checkChromeExtensions(profileExtensions = []) {
+    if (!(Array.isArray(profileExtensions) && profileExtensions.length)) {
+      return [];
+    }
+    const extensionsToDownload = this.#getExtensionsToDownload(profileExtensions);
+    const downloadedArchives = await this.downloadChromeExtensions(extensionsToDownload);
+    const filteredArchives = downloadedArchives.filter(Boolean);
+    if (filteredArchives.length) {
+      const [downloadedFolders] = filteredArchives.map(archivePath => archivePath.split(_path.sep).reverse());
+      this.#existedChromeExtensions = [...this.#existedChromeExtensions, ...downloadedFolders];
+      const promises = (0, _common.composeExtractionPromises)(filteredArchives);
+      await Promise.all(promises);
+    }
+    return this.getExtensionsStrToIncludeAsOrbitaParam(profileExtensions);
+  }
+  #getExtensionsToDownload(profileExtensions) {
+    const existedExtensionsFolders = [...this.#existedChromeExtensions, ...this.existedUserExtensions];
+    const existedOriginalIds = existedExtensionsFolders.map(val => {
+      const [originalId] = val.split('@');
+      return originalId;
+    });
+    return profileExtensions.reduce((res, val) => {
+      const [originalId] = val.split('@');
+      const extensionExists = existedOriginalIds.includes(originalId);
+      if (!extensionExists) {
+        res.push(val);
+      }
+      return res;
+    }, []);
+  }
+  async downloadChromeExtensions(idsToDownload = []) {
+    if (!(Array.isArray(idsToDownload) && idsToDownload.length)) {
+      return [];
+    }
+    const promises = idsToDownload.map(async id => {
+      const [originalId] = id.split('@');
+      const extUrl = EXTENSION_URL.replace('{ext_id}', originalId);
+      const uploadedProfileMetadata = await getExtMetadata(extUrl);
+      const reqPath = uploadedProfileMetadata.req.path;
+      const extVer = getExtVersion(reqPath);
+      const buffer = await new Promise(res => {
+        const chunks = [];
+        _requestretry.default.get(extUrl, {
+          maxAttempts: 3,
+          retryDelay: 1000,
+          timeout: 8 * 1000,
+          fullResponse: false
+        }).on('data', data => chunks.push(data)).on('end', () => res(Buffer.concat(chunks)));
+      });
+      let zipExt;
+      try {
+        zipExt = crxToZip(buffer);
+      } catch (e) {
+        console.log(e);
+        return '';
+      }
+      const archiveZipPath = (0, _path.join)(_common.CHROME_EXTENSIONS_PATH, originalId + '@' + extVer + '.zip');
+      const archiveZip = (0, _fs.createWriteStream)(archiveZipPath);
+      archiveZip.write(zipExt);
+      archiveZip.close();
+      return new Promise(r => archiveZip.on('close', () => r(archiveZipPath)));
+    });
+    return Promise.all(promises);
+  }
+  async getExtensionsPolicies() {
+    const globalExtConfig = await _requestretry.default.get(`${this.apiBaseUrl}/gologin-settings/chrome_ext_policies`, {
+      headers: {
+        Authorization: `Bearer ${this.accessToken}`,
+        'user-agent': this.userAgent,
+        'x-two-factor-token': this.twoFaKey || ''
+      },
+      json: true,
+      maxAttempts: 2,
+      retryDelay: 1000,
+      timeout: 10 * 1000,
+      fullResponse: false
+    });
+    const chromeExtPolicies = globalExtConfig?.chromeExtPolicies || {};
+    const {
+      useLocalExtStorage = false,
+      deleteProfileExtFolders = false,
+      useCookiesExt = true
+    } = chromeExtPolicies;
+    this.#useLocalExtStorage = useLocalExtStorage;
+    this.#deleteProfileExtFolders = deleteProfileExtFolders;
+    this.#useCookiesExt = useCookiesExt;
+  }
+  async updateExtensions() {
+    const fileList = await readdir(_common.CHROME_EXTENSIONS_PATH).catch(() => []);
+    if (!fileList.length) {
+      return;
+    }
+    const oldFolders = [];
+    const versionCheckPromises = fileList.map(async extension => {
+      if (!extension.includes('@') || extension.includes('.zip')) {
+        return '';
+      }
+      const [originalId, currentVersion] = extension.split('@');
+      const extUrl = EXTENSION_URL.replace('{ext_id}', originalId);
+      const uploadedProfileMetadata = await getExtMetadata(extUrl);
+      const reqPath = uploadedProfileMetadata.req.path;
+      const availableVersion = getExtVersion(reqPath);
+      if (currentVersion === availableVersion) {
+        return '';
+      }
+      oldFolders.push((0, _path.join)(_common.CHROME_EXTENSIONS_PATH, extension));
+      return originalId;
+    });
+    const extensionsNames = (await Promise.all(versionCheckPromises)).filter(Boolean);
+    const archivesPaths = (await this.downloadChromeExtensions(extensionsNames)).filter(Boolean);
+    const extractionPromises = (0, _common.composeExtractionPromises)(archivesPaths);
+    await Promise.all(extractionPromises);
+    const removeFoldersPromises = oldFolders.map(folder => rmdir(folder, {
+      recursive: true,
+      maxRetries: 3
+    }).catch(() => {}));
+    await Promise.all(removeFoldersPromises).then(() => this.#extensionsUpdating = false);
+  }
+  async checkLocalExtensions() {
+    if (this.#extensionsUpdating || !this.accessToken) {
+      return;
+    }
+    const fileList = await readdir(_common.CHROME_EXTENSIONS_PATH).catch(() => []);
+    if (!fileList.length) {
+      return;
+    }
+    const extensionsIds = fileList.filter(folderName => folderName.includes('@') && !folderName.includes('.zip')).map(folderName => {
+      const [name] = folderName.split('@');
+      return name;
+    });
+    if (!extensionsIds.length) {
+      return;
+    }
+    this.insertExtensionsToDb(extensionsIds);
+  }
+  async insertExtensionsToDb(extensionsIds, pathToExtensions = _common.CHROME_EXTENSIONS_PATH) {
+    if (!extensionsIds?.length) {
+      return;
+    }
+    const checkResponse = await (0, _requestretry.default)(`${this.apiBaseUrl}/extensions/check`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.accessToken}`,
+        'user-agent': this.userAgent,
+        'x-two-factor-token': this.twoFaKey || ''
+      },
+      body: {
+        extensionsIds
+      },
+      json: true
+    });
+    const {
+      extensionsToAdd = []
+    } = checkResponse.body;
+    if (!extensionsToAdd.length) {
+      return;
+    }
+    const extensionsToUpdate = await this.getExtensionsNameAndImage(extensionsToAdd, pathToExtensions);
+    (0, _requestretry.default)(`${this.apiBaseUrl}/extensions/create`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.accessToken}`,
+        'user-agent': this.userAgent,
+        'x-two-factor-token': this.twoFaKey || ''
+      },
+      body: {
+        extensionsInfo: extensionsToUpdate
+      },
+      json: true
+    });
+  }
+  getExtensionsToInstall(extensionsFromPref, extensionsFromDB) {
+    if (!extensionsFromPref) {
+      return [];
+    }
+    const objectEntries = Object.entries(extensionsFromPref);
+    const extensionsInPref = objectEntries?.map(([_, settings]) => {
+      const [extFolderName] = settings.path.split(_path.sep).reverse();
+      const [originalId] = extFolderName.split('@');
+      return originalId;
+    }) || [];
+    return extensionsFromDB.reduce((acc, extension) => {
+      const [extFolderName] = extension.split(_path.sep).reverse();
+      const [originalId] = extFolderName.split('@');
+      if (!extensionsInPref.includes(originalId)) {
+        acc.push(extension);
+      }
+      return acc;
+    }, []);
+  }
+}
+exports.ExtensionsManager = ExtensionsManager;
+const crxToZip = buf => {
+  if (buf[0] === 80 && buf[1] === 75 && buf[2] === 3 && buf[3] === 4) {
+    return buf;
+  }
+  if (!(buf[0] === 67 || buf[1] === 114 || buf[2] === 50 || buf[3] === 52)) {
+    throw new Error('Invalid header: Does not start with Cr24');
+  }
+  const isV3 = buf[4] === 3;
+  const isV2 = buf[4] === 2;
+  if (!(isV2 || isV3) || buf[5] || buf[6] || buf[7]) {
+    throw new Error('Unexpected crx format version number.');
+  }
+  if (isV2) {
+    const publicKeyLength = calcLength(buf[8], buf[9], buf[10], buf[11]);
+    const signatureLength = calcLength(buf[12], buf[13], buf[14], buf[15]);
+    const zipStartOffset = 16 + publicKeyLength + signatureLength;
+    return buf.slice(zipStartOffset, buf.length);
+  }
+  const headerSize = calcLength(buf[8], buf[9], buf[10], buf[11]);
+  const zipStartOffset = 12 + headerSize;
+  return buf.slice(zipStartOffset, buf.length);
+};
+const calcLength = (a, b, c, d) => {
+  let length = 0;
+  length += a << 0;
+  length += b << 8;
+  length += c << 16;
+  length += d << 24 >>> 0;
+  return length;
+};
+const getExtMetadata = extUrl => _requestretry.default.head(extUrl, {
+  maxAttempts: 3,
+  retryDelay: 2000,
+  timeout: 2 * 1000,
+  fullResponse: true
+});
+const getExtVersion = metadata => {
+  const [extFullName = ''] = metadata.split('/').reverse();
+  const [extName = ''] = extFullName.split('.');
+  const splitExtName = extName.split('_');
+  splitExtName.shift();
+  return splitExtName.join('_');
+};
+var _default = exports.default = ExtensionsManager;