gologin 1.0.35 → 1.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gologin.js +59 -34
- package/package.json +1 -3
- package/selenium/chromedriver +0 -0
- package/selenium/chromedriver.exe +0 -0
- package/selenium/example.js +0 -32
- package/selenium/gologin-create-profile.py +0 -30
- package/selenium/gologin-local.py +0 -30
- package/selenium/gologin-selenium-multiprocess.py +0 -50
- package/selenium/gologin-selenium.py +0 -28
- package/selenium/gologin.py +0 -458
- package/selenium/mac/chromedriver +0 -0
package/gologin.js
CHANGED
|
@@ -123,7 +123,17 @@ class GoLogin {
|
|
|
123
123
|
'Authorization': `Bearer ${this.access_token}`
|
|
124
124
|
}
|
|
125
125
|
})
|
|
126
|
-
debug(profileResponse.body);
|
|
126
|
+
debug("profileResponse", profileResponse.statusCode, profileResponse.body);
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
if (profileResponse.statusCode === 404) {
|
|
130
|
+
throw new Error(JSON.parse(profileResponse.body).message);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (profileResponse.statusCode === 403) {
|
|
134
|
+
throw new Error(JSON.parse(profileResponse.body).message);
|
|
135
|
+
}
|
|
136
|
+
|
|
127
137
|
if (profileResponse.statusCode !== 200) {
|
|
128
138
|
throw new Error(`Gologin /browser/${id} response error ${profileResponse.statusCode} INVALID TOKEN OR PROFILE NOT FOUND`);
|
|
129
139
|
}
|
|
@@ -132,6 +142,7 @@ class GoLogin {
|
|
|
132
142
|
throw new Error("invalid token");
|
|
133
143
|
}
|
|
134
144
|
|
|
145
|
+
|
|
135
146
|
return JSON.parse(profileResponse.body);
|
|
136
147
|
}
|
|
137
148
|
|
|
@@ -537,14 +548,22 @@ class GoLogin {
|
|
|
537
548
|
let data = null;
|
|
538
549
|
if (proxy!==null && proxy.mode !== "none") {
|
|
539
550
|
if (proxy.mode.includes('socks')) {
|
|
540
|
-
|
|
551
|
+
for(let i=0; i<5; i++){
|
|
552
|
+
try{
|
|
553
|
+
debug('getting timeZone socks try', i+1);
|
|
554
|
+
return this.getTimezoneWithSocks(proxy);
|
|
555
|
+
} catch (e) {
|
|
556
|
+
console.log(e.message);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
throw new Error(`Socks proxy connection timed out`);
|
|
541
560
|
}
|
|
542
561
|
|
|
543
562
|
const proxyUrl = `${proxy.mode}://${proxy.username}:${proxy.password}@${proxy.host}:${proxy.port}`;
|
|
544
|
-
debug('getTimeZone start https://time.gologin.com', proxyUrl);
|
|
545
|
-
data = await requests.get('https://time.gologin.com', { proxy: proxyUrl, timeout: 20 * 1000, maxAttempts: 5 });
|
|
563
|
+
debug('getTimeZone start https://time.gologin.com/timezone', proxyUrl);
|
|
564
|
+
data = await requests.get('https://time.gologin.com/timezone', { proxy: proxyUrl, timeout: 20 * 1000, maxAttempts: 5 });
|
|
546
565
|
} else {
|
|
547
|
-
data = await requests.get('https://time.gologin.com', { timeout: 20 * 1000, maxAttempts: 5 });
|
|
566
|
+
data = await requests.get('https://time.gologin.com/timezone', { timeout: 20 * 1000, maxAttempts: 5 });
|
|
548
567
|
}
|
|
549
568
|
debug('getTimeZone finish', data.body);
|
|
550
569
|
this._tz = JSON.parse(data.body);
|
|
@@ -1128,47 +1147,53 @@ class GoLogin {
|
|
|
1128
1147
|
|
|
1129
1148
|
async startRemote(delay_ms = 10000) {
|
|
1130
1149
|
debug(`startRemote ${this.profile_id}`);
|
|
1150
|
+
|
|
1151
|
+
/*
|
|
1152
|
+
if (profileResponse.statusCode !== 202) {
|
|
1153
|
+
return {'status': 'failure', 'code': profileResponse.statusCode};
|
|
1154
|
+
}
|
|
1155
|
+
*/
|
|
1156
|
+
|
|
1157
|
+
// if (profileResponse.body === 'ok') {
|
|
1158
|
+
const profile = await this.getProfile();
|
|
1159
|
+
|
|
1131
1160
|
const profileResponse = await requests.post(`https://api.gologin.com/browser/${this.profile_id}/web`, {
|
|
1132
1161
|
headers: {
|
|
1133
1162
|
'Authorization': `Bearer ${this.access_token}`
|
|
1134
1163
|
}
|
|
1135
1164
|
});
|
|
1165
|
+
|
|
1166
|
+
debug('profileResponse', profileResponse.statusCode, profileResponse.body);
|
|
1136
1167
|
|
|
1137
1168
|
if (profileResponse.statusCode === 401){
|
|
1138
1169
|
throw new Error("invalid token");
|
|
1139
1170
|
}
|
|
1140
1171
|
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
} = navigator;
|
|
1162
|
-
this.language = language;
|
|
1163
|
-
const [screenWidth, screenHeight] = resolution.split('x');
|
|
1164
|
-
this.resolution = {
|
|
1165
|
-
width: parseInt(screenWidth, 10),
|
|
1166
|
-
height: parseInt(screenHeight, 10),
|
|
1167
|
-
};
|
|
1172
|
+
const { navigator = {}, fonts, os: profileOs } = profile;
|
|
1173
|
+
this.fontsMasking = fonts?.enableMasking;
|
|
1174
|
+
this.profileOs = profileOs;
|
|
1175
|
+
this.differentOs =
|
|
1176
|
+
profileOs !== 'android' && (
|
|
1177
|
+
OS_PLATFORM === 'win32' && profileOs !== 'win' ||
|
|
1178
|
+
OS_PLATFORM === 'darwin' && profileOs !== 'mac' ||
|
|
1179
|
+
OS_PLATFORM === 'linux' && profileOs !== 'lin'
|
|
1180
|
+
);
|
|
1181
|
+
|
|
1182
|
+
const {
|
|
1183
|
+
resolution = '1920x1080',
|
|
1184
|
+
language = 'en-US,en;q=0.9',
|
|
1185
|
+
} = navigator;
|
|
1186
|
+
this.language = language;
|
|
1187
|
+
const [screenWidth, screenHeight] = resolution.split('x');
|
|
1188
|
+
this.resolution = {
|
|
1189
|
+
width: parseInt(screenWidth, 10),
|
|
1190
|
+
height: parseInt(screenHeight, 10),
|
|
1191
|
+
};
|
|
1168
1192
|
|
|
1169
|
-
|
|
1193
|
+
let wsUrl = await this.waitDebuggingUrl(delay_ms);
|
|
1194
|
+
if(wsUrl!=''){
|
|
1170
1195
|
return { 'status': 'success', wsUrl }
|
|
1171
|
-
}
|
|
1196
|
+
}
|
|
1172
1197
|
|
|
1173
1198
|
return { 'status': 'failure', 'message': profileResponse.body };
|
|
1174
1199
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gologin",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.38",
|
|
4
4
|
"description": "A high-level API to control Orbita browser over GoLogin API",
|
|
5
5
|
"main": "./gologin.js",
|
|
6
6
|
"repository": {
|
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
"dependencies": {
|
|
16
16
|
"archiver": "^3.1.1",
|
|
17
17
|
"child_process": "^1.0.2",
|
|
18
|
-
"chromedriver": "^89.0.0",
|
|
19
18
|
"decompress": "^4.2.1",
|
|
20
19
|
"decompress-unzip": "^4.0.1",
|
|
21
20
|
"form-data": "^3.0.0",
|
|
@@ -25,7 +24,6 @@
|
|
|
25
24
|
"request": "^2.88.2",
|
|
26
25
|
"requestretry": "^4.1.0",
|
|
27
26
|
"rimraf": "^3.0.2",
|
|
28
|
-
"selenium-webdriver": "^4.0.0-alpha.7",
|
|
29
27
|
"simple-proxy-agent": "^1.1.0",
|
|
30
28
|
"sqlite": "^4.0.23",
|
|
31
29
|
"sqlite3": "^5.0.2",
|
package/selenium/chromedriver
DELETED
|
Binary file
|
|
Binary file
|
package/selenium/example.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
const GoLogin = require('../gologin');
|
|
2
|
-
const webdriver = require("selenium-webdriver");
|
|
3
|
-
const chrome = require("selenium-webdriver/chrome");
|
|
4
|
-
chrome.setDefaultService(new chrome.ServiceBuilder('./chromedriver').build());
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
(async () =>{
|
|
8
|
-
const GL = new GoLogin({
|
|
9
|
-
token: 'yU0token',
|
|
10
|
-
profile_id: 'yU0Pr0f1leiD',
|
|
11
|
-
executablePath: '/usr/bin/orbita-browser/chrome',
|
|
12
|
-
});
|
|
13
|
-
console.log('creating startup')
|
|
14
|
-
await GL.createStartup();
|
|
15
|
-
console.log('spawn arguments')
|
|
16
|
-
const arguments = await GL.spawnArguments();
|
|
17
|
-
console.log('set options')
|
|
18
|
-
const chromeOptions = new chrome.Options();
|
|
19
|
-
arguments.forEach((e) => {
|
|
20
|
-
console.log('e=', e);
|
|
21
|
-
chromeOptions.addArguments(e)
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
chromeOptions.setChromeBinaryPath('/usr/bin/orbita-browser/chrome');
|
|
25
|
-
|
|
26
|
-
driver = new webdriver.Builder()
|
|
27
|
-
.forBrowser("chrome")
|
|
28
|
-
.setChromeOptions(chromeOptions)
|
|
29
|
-
.build();
|
|
30
|
-
|
|
31
|
-
await driver.get('https://myip.link')
|
|
32
|
-
})();
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
from gologin import GoLogin
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
gl = GoLogin({
|
|
5
|
-
"token": "yU0token",
|
|
6
|
-
})
|
|
7
|
-
|
|
8
|
-
profile_id = gl.create({
|
|
9
|
-
"name": 'profile_mac',
|
|
10
|
-
"os": 'mac',
|
|
11
|
-
"navigator": {
|
|
12
|
-
"language": 'enUS',
|
|
13
|
-
"userAgent": 'MyUserAgent',
|
|
14
|
-
"resolution": '1024x768',
|
|
15
|
-
"platform": 'mac',
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
print('profile id=', profile_id);
|
|
20
|
-
|
|
21
|
-
gl.update({
|
|
22
|
-
"id": profile_id,
|
|
23
|
-
"name": 'profile_mac2',
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
profile = gl.getProfile(profile_id);
|
|
27
|
-
|
|
28
|
-
print('new profile name=', profile.get("name"));
|
|
29
|
-
|
|
30
|
-
gl.delete(profile_id)
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
from sys import platform
|
|
3
|
-
from selenium import webdriver
|
|
4
|
-
from selenium.webdriver.chrome.options import Options
|
|
5
|
-
from gologin import GoLogin
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
gl = GoLogin({
|
|
9
|
-
"token": "yU0token",
|
|
10
|
-
"profile_id": "yU0Pr0f1leiD",
|
|
11
|
-
"local": True,
|
|
12
|
-
"credentials_enable_service": False,
|
|
13
|
-
})
|
|
14
|
-
|
|
15
|
-
if platform == "linux" or platform == "linux2":
|
|
16
|
-
chrome_driver_path = "./chromedriver"
|
|
17
|
-
elif platform == "darwin":
|
|
18
|
-
chrome_driver_path = "./mac/chromedriver"
|
|
19
|
-
elif platform == "win32":
|
|
20
|
-
chrome_driver_path = "chromedriver.exe"
|
|
21
|
-
|
|
22
|
-
debugger_address = gl.start()
|
|
23
|
-
chrome_options = Options()
|
|
24
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
25
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
26
|
-
driver.get("http://www.python.org")
|
|
27
|
-
assert "Python" in driver.title
|
|
28
|
-
driver.close()
|
|
29
|
-
time.sleep(3)
|
|
30
|
-
gl.stop()
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
import os
|
|
3
|
-
from multiprocessing import Pool
|
|
4
|
-
from sys import platform
|
|
5
|
-
from selenium import webdriver
|
|
6
|
-
from selenium.webdriver.chrome.options import Options
|
|
7
|
-
from gologin import GoLogin
|
|
8
|
-
|
|
9
|
-
def scrap(profile):
|
|
10
|
-
gl = GoLogin({
|
|
11
|
-
'token': 'yU0token',
|
|
12
|
-
'profile_id': profile['profile_id'],
|
|
13
|
-
'port': profile['port'],
|
|
14
|
-
})
|
|
15
|
-
|
|
16
|
-
if platform == "linux" or platform == "linux2":
|
|
17
|
-
chrome_driver_path = './chromedriver'
|
|
18
|
-
elif platform == "darwin":
|
|
19
|
-
chrome_driver_path = './mac/chromedriver'
|
|
20
|
-
elif platform == "win32":
|
|
21
|
-
chrome_driver_path = 'chromedriver.exe'
|
|
22
|
-
|
|
23
|
-
debugger_address = gl.start()
|
|
24
|
-
chrome_options = Options()
|
|
25
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
26
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
27
|
-
driver.get("http://www.python.org")
|
|
28
|
-
print('ready', profile['profile_id'], driver.title)
|
|
29
|
-
time.sleep(10)
|
|
30
|
-
print('closing', profile['profile_id'])
|
|
31
|
-
driver.close()
|
|
32
|
-
gl.stop()
|
|
33
|
-
|
|
34
|
-
profiles = [
|
|
35
|
-
{'profile_id': 'profile_id_1', 'port': 3500},
|
|
36
|
-
{'profile_id': 'profile_id_2', 'port': 3501},
|
|
37
|
-
{'profile_id': 'profile_id_3', 'port': 3502},
|
|
38
|
-
]
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
with Pool(3) as p:
|
|
42
|
-
p.map(scrap, profiles)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
if platform == "win32":
|
|
46
|
-
os.system('taskkill /im chrome.exe /f')
|
|
47
|
-
os.system('taskkill /im chromedriver.exe /f')
|
|
48
|
-
else:
|
|
49
|
-
os.system('killall -9 chrome')
|
|
50
|
-
os.system('killall -9 chromedriver')
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
from sys import platform
|
|
3
|
-
from selenium import webdriver
|
|
4
|
-
from selenium.webdriver.chrome.options import Options
|
|
5
|
-
from gologin import GoLogin
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
gl = GoLogin({
|
|
9
|
-
"token": "yU0token",
|
|
10
|
-
"profile_id": "yU0Pr0f1leiD",
|
|
11
|
-
})
|
|
12
|
-
|
|
13
|
-
if platform == "linux" or platform == "linux2":
|
|
14
|
-
chrome_driver_path = "./chromedriver"
|
|
15
|
-
elif platform == "darwin":
|
|
16
|
-
chrome_driver_path = "./mac/chromedriver"
|
|
17
|
-
elif platform == "win32":
|
|
18
|
-
chrome_driver_path = "chromedriver.exe"
|
|
19
|
-
|
|
20
|
-
debugger_address = gl.start()
|
|
21
|
-
chrome_options = Options()
|
|
22
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
23
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
24
|
-
driver.get("http://www.python.org")
|
|
25
|
-
assert "Python" in driver.title
|
|
26
|
-
driver.close()
|
|
27
|
-
time.sleep(3)
|
|
28
|
-
gl.stop()
|
package/selenium/gologin.py
DELETED
|
@@ -1,458 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import time
|
|
3
|
-
import os
|
|
4
|
-
import stat
|
|
5
|
-
import sys
|
|
6
|
-
import shutil
|
|
7
|
-
import requests
|
|
8
|
-
import zipfile
|
|
9
|
-
import subprocess
|
|
10
|
-
import pathlib
|
|
11
|
-
import tempfile
|
|
12
|
-
|
|
13
|
-
API_URL = 'https://api.gologin.com'
|
|
14
|
-
|
|
15
|
-
class GoLogin(object):
|
|
16
|
-
def __init__(self, options):
|
|
17
|
-
self.access_token = options.get('token')
|
|
18
|
-
|
|
19
|
-
self.tmpdir = options.get('tmpdir', tempfile.gettempdir())
|
|
20
|
-
self.address = options.get('address', '127.0.0.1')
|
|
21
|
-
self.extra_params = options.get('extra_params', [])
|
|
22
|
-
self.port = options.get('port', 3500)
|
|
23
|
-
self.local = options.get('local', False)
|
|
24
|
-
self.spawn_browser = options.get('spawn_browser', True)
|
|
25
|
-
self.credentials_enable_service = options.get('credentials_enable_service')
|
|
26
|
-
|
|
27
|
-
home = str(pathlib.Path.home())
|
|
28
|
-
self.executablePath = options.get('executablePath', os.path.join(home, '.gologin/browser/orbita-browser/chrome'))
|
|
29
|
-
print('executablePath', self.executablePath)
|
|
30
|
-
if self.extra_params:
|
|
31
|
-
print('extra_params', self.extra_params)
|
|
32
|
-
self.setProfileId(options.get('profile_id'))
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def setProfileId(self, profile_id):
|
|
36
|
-
self.profile_id = profile_id
|
|
37
|
-
if self.profile_id==None:
|
|
38
|
-
return
|
|
39
|
-
self.profile_path = os.path.join(self.tmpdir, 'gologin_'+self.profile_id)
|
|
40
|
-
self.profile_zip_path = os.path.join(self.tmpdir, 'gologin_'+self.profile_id+'.zip')
|
|
41
|
-
self.profile_zip_path_upload = os.path.join(self.tmpdir, 'gologin_'+self.profile_id+'_upload.zip')
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def spawnBrowser(self):
|
|
45
|
-
proxy = self.proxy
|
|
46
|
-
proxy_host = ''
|
|
47
|
-
if proxy:
|
|
48
|
-
if proxy.get('mode')==None or proxy.get('mode')=='geolocation':
|
|
49
|
-
proxy['mode'] = 'http'
|
|
50
|
-
proxy_host = proxy.get('host')
|
|
51
|
-
proxy = self.formatProxyUrl(proxy)
|
|
52
|
-
|
|
53
|
-
tz = self.tz.get('timezone')
|
|
54
|
-
|
|
55
|
-
params = [
|
|
56
|
-
self.executablePath,
|
|
57
|
-
'--remote-debugging-port='+str(self.port),
|
|
58
|
-
'--user-data-dir='+self.profile_path,
|
|
59
|
-
'--password-store=basic',
|
|
60
|
-
'--tz='+tz,
|
|
61
|
-
'--gologin-profile='+self.profile_name,
|
|
62
|
-
'--lang=en',
|
|
63
|
-
]
|
|
64
|
-
if proxy:
|
|
65
|
-
hr_rules = '"MAP * 0.0.0.0 , EXCLUDE %s"'%(proxy_host)
|
|
66
|
-
params.append('--proxy-server='+proxy)
|
|
67
|
-
params.append('--host-resolver-rules='+hr_rules)
|
|
68
|
-
|
|
69
|
-
for param in self.extra_params:
|
|
70
|
-
params.append(param)
|
|
71
|
-
|
|
72
|
-
if sys.platform == "darwin":
|
|
73
|
-
subprocess.Popen(params)
|
|
74
|
-
else:
|
|
75
|
-
subprocess.Popen(params, start_new_session=True)
|
|
76
|
-
|
|
77
|
-
try_count = 1
|
|
78
|
-
url = str(self.address) + ':' + str(self.port)
|
|
79
|
-
while try_count<100:
|
|
80
|
-
try:
|
|
81
|
-
data = requests.get('http://'+url+'/json').content
|
|
82
|
-
break
|
|
83
|
-
except:
|
|
84
|
-
try_count += 1
|
|
85
|
-
time.sleep(1)
|
|
86
|
-
|
|
87
|
-
return url
|
|
88
|
-
|
|
89
|
-
def start(self):
|
|
90
|
-
profile_path = self.createStartup()
|
|
91
|
-
if self.spawn_browser == True:
|
|
92
|
-
return self.spawnBrowser()
|
|
93
|
-
return profile_path
|
|
94
|
-
|
|
95
|
-
def zipdir(self, path, ziph):
|
|
96
|
-
for root, dirs, files in os.walk(path):
|
|
97
|
-
for file in files:
|
|
98
|
-
path = os.path.join(root, file)
|
|
99
|
-
if not os.path.exists(path):
|
|
100
|
-
continue
|
|
101
|
-
if stat.S_ISSOCK(os.stat(path).st_mode):
|
|
102
|
-
continue
|
|
103
|
-
try:
|
|
104
|
-
ziph.write(path, path.replace(self.profile_path, ''))
|
|
105
|
-
except:
|
|
106
|
-
continue
|
|
107
|
-
|
|
108
|
-
def stop(self):
|
|
109
|
-
self.sanitizeProfile()
|
|
110
|
-
if self.local==False:
|
|
111
|
-
self.commitProfile()
|
|
112
|
-
os.remove(self.profile_zip_path_upload)
|
|
113
|
-
shutil.rmtree(self.profile_path)
|
|
114
|
-
|
|
115
|
-
def commitProfile(self):
|
|
116
|
-
zipf = zipfile.ZipFile(self.profile_zip_path_upload, 'w', zipfile.ZIP_DEFLATED)
|
|
117
|
-
self.zipdir(self.profile_path, zipf)
|
|
118
|
-
zipf.close()
|
|
119
|
-
|
|
120
|
-
headers = {
|
|
121
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
122
|
-
'User-Agent': 'Selenium-API'
|
|
123
|
-
}
|
|
124
|
-
# print('profile size=', os.stat(self.profile_zip_path_upload).st_size)
|
|
125
|
-
|
|
126
|
-
signedUrl = requests.get(API_URL + '/browser/' + self.profile_id + '/storage-signature', headers=headers).content.decode('utf-8')
|
|
127
|
-
|
|
128
|
-
requests.put(signedUrl, data=open(self.profile_zip_path_upload, 'rb'))
|
|
129
|
-
|
|
130
|
-
# print('commit profile complete')
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def sanitizeProfile(self):
|
|
134
|
-
remove_dirs = [
|
|
135
|
-
'Default/Cache',
|
|
136
|
-
'Default/Service Worker/CacheStorage',
|
|
137
|
-
'Default/Code Cache',
|
|
138
|
-
'Default/GPUCache',
|
|
139
|
-
'GrShaderCache',
|
|
140
|
-
'ShaderCache',
|
|
141
|
-
'biahpgbdmdkfgndcmfiipgcebobojjkp',
|
|
142
|
-
'afalakplffnnnlkncjhbmahjfjhmlkal',
|
|
143
|
-
'cffkpbalmllkdoenhmdmpbkajipdjfam',
|
|
144
|
-
'Dictionaries',
|
|
145
|
-
'enkheaiicpeffbfgjiklngbpkilnbkoi',
|
|
146
|
-
'oofiananboodjbbmdelgdommihjbkfag',
|
|
147
|
-
'SafetyTips',
|
|
148
|
-
'fonts',
|
|
149
|
-
];
|
|
150
|
-
|
|
151
|
-
for d in remove_dirs:
|
|
152
|
-
fpath = os.path.join(self.profile_path, d)
|
|
153
|
-
if os.path.exists(fpath):
|
|
154
|
-
try:
|
|
155
|
-
shutil.rmtree(fpath)
|
|
156
|
-
except:
|
|
157
|
-
continue
|
|
158
|
-
|
|
159
|
-
def formatProxyUrl(self, proxy):
|
|
160
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
161
|
-
|
|
162
|
-
def formatProxyUrlPassword(self, proxy):
|
|
163
|
-
if proxy.get('username', '')=='':
|
|
164
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
165
|
-
else:
|
|
166
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('username','')+':'+proxy.get('password')+'@'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def getTimeZone(self):
|
|
170
|
-
proxy = self.proxy
|
|
171
|
-
if proxy:
|
|
172
|
-
proxies = {proxy.get('mode'): self.formatProxyUrlPassword(proxy)}
|
|
173
|
-
data = requests.get('https://time.gologin.com', proxies=proxies)
|
|
174
|
-
else:
|
|
175
|
-
data = requests.get('https://time.gologin.com')
|
|
176
|
-
return json.loads(data.content.decode('utf-8'))
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
def getProfile(self, profile_id=None):
|
|
180
|
-
profile = self.profile_id if profile_id==None else profile_id
|
|
181
|
-
headers = {
|
|
182
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
183
|
-
'User-Agent': 'Selenium-API'
|
|
184
|
-
}
|
|
185
|
-
return json.loads(requests.get(API_URL + '/browser/' + profile, headers=headers).content.decode('utf-8'))
|
|
186
|
-
|
|
187
|
-
def downloadProfileZip(self):
|
|
188
|
-
s3path = self.profile.get('s3Path', '')
|
|
189
|
-
data = ''
|
|
190
|
-
if s3path=='':
|
|
191
|
-
# print('downloading profile direct')
|
|
192
|
-
headers = {
|
|
193
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
194
|
-
'User-Agent': 'Selenium-API'
|
|
195
|
-
}
|
|
196
|
-
data = requests.get(API_URL + '/browser/'+self.profile_id, headers=headers).content
|
|
197
|
-
else:
|
|
198
|
-
# print('downloading profile s3')
|
|
199
|
-
s3url = 'https://gprofiles.gologin.com/' + s3path.replace(' ', '+')
|
|
200
|
-
data = requests.get(s3url).content
|
|
201
|
-
|
|
202
|
-
if len(data)==0:
|
|
203
|
-
self.createEmptyProfile()
|
|
204
|
-
else:
|
|
205
|
-
with open(self.profile_zip_path, 'wb') as f:
|
|
206
|
-
f.write(data)
|
|
207
|
-
|
|
208
|
-
try:
|
|
209
|
-
self.extractProfileZip()
|
|
210
|
-
except:
|
|
211
|
-
self.createEmptyProfile()
|
|
212
|
-
self.extractProfileZip()
|
|
213
|
-
|
|
214
|
-
if not os.path.exists(os.path.join(self.profile_path, 'Default/Preferences')):
|
|
215
|
-
self.createEmptyProfile()
|
|
216
|
-
self.extractProfileZip()
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def createEmptyProfile(self):
|
|
220
|
-
print('createEmptyProfile')
|
|
221
|
-
empty_profile = '../gologin_zeroprofile.zip'
|
|
222
|
-
if not os.path.exists(empty_profile):
|
|
223
|
-
empty_profile = 'gologin_zeroprofile.zip'
|
|
224
|
-
shutil.copy(empty_profile, self.profile_zip_path)
|
|
225
|
-
|
|
226
|
-
def extractProfileZip(self):
|
|
227
|
-
with zipfile.ZipFile(self.profile_zip_path, 'r') as zip_ref:
|
|
228
|
-
zip_ref.extractall(self.profile_path)
|
|
229
|
-
os.remove(self.profile_zip_path)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
def getGeolocationParams(self, profileGeolocationParams, tzGeolocationParams):
|
|
233
|
-
if profileGeolocationParams.get('fillBasedOnIp'):
|
|
234
|
-
return {
|
|
235
|
-
'mode': profileGeolocationParams['mode'],
|
|
236
|
-
'latitude': float(tzGeolocationParams['latitude']),
|
|
237
|
-
'longitude': float(tzGeolocationParams['longitude']),
|
|
238
|
-
'accuracy': float(tzGeolocationParams['accuracy']),
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
return {
|
|
242
|
-
'mode': profileGeolocationParams['mode'],
|
|
243
|
-
'latitude': profileGeolocationParams['latitude'],
|
|
244
|
-
'longitude': profileGeolocationParams['longitude'],
|
|
245
|
-
'accuracy': profileGeolocationParams['accuracy'],
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def convertPreferences(self, preferences):
|
|
250
|
-
resolution = preferences.get('resolution', '1920x1080')
|
|
251
|
-
preferences['screenWidth'] = int(resolution.split('x')[0])
|
|
252
|
-
preferences['screenHeight'] = int(resolution.split('x')[1])
|
|
253
|
-
|
|
254
|
-
self.tz = self.getTimeZone()
|
|
255
|
-
# print('tz=', self.tz)
|
|
256
|
-
tzGeoLocation = {
|
|
257
|
-
'latitude': self.tz.get('ll', [0, 0])[0],
|
|
258
|
-
'longitude': self.tz.get('ll', [0, 0])[1],
|
|
259
|
-
'accuracy': self.tz.get('accuracy', 0),
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
preferences['geoLocation'] = self.getGeolocationParams(preferences['geolocation'], tzGeoLocation)
|
|
263
|
-
|
|
264
|
-
preferences['webRtc'] = {
|
|
265
|
-
'mode': 'public' if preferences.get('webRTC',{}).get('mode') == 'alerted' else preferences.get('webRTC',{}).get('mode'),
|
|
266
|
-
'publicIP': self.tz['ip'] if preferences.get('webRTC',{}).get('fillBasedOnIp') else preferences.get('webRTC',{}).get('publicIp'),
|
|
267
|
-
'localIps': preferences.get('webRTC',{}).get('localIps', [])
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
preferences['timezone'] = {
|
|
271
|
-
'id': self.tz.get('timezone')
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
preferences['webgl_noise_value'] = preferences.get('webGL', {}).get('noise')
|
|
275
|
-
preferences['get_client_rects_noise'] = preferences.get('webGL', {}).get('getClientRectsNoise')
|
|
276
|
-
preferences['canvasMode'] = preferences.get('canvas', {}).get('mode')
|
|
277
|
-
preferences['canvasNoise'] = preferences.get('canvas', {}).get('noise')
|
|
278
|
-
preferences['audioContext'] = {
|
|
279
|
-
'enable': preferences.get('audioContext').get('mode', 'off'),
|
|
280
|
-
'noiseValue': preferences.get('audioContext').get('noise'),
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
preferences['webgl'] = {
|
|
284
|
-
'metadata': {
|
|
285
|
-
'vendor': preferences.get('webGLMetadata', {}).get('vendor'),
|
|
286
|
-
'renderer': preferences.get('webGLMetadata', {}).get('renderer'),
|
|
287
|
-
'mode': preferences.get('webGLMetadata', {}).get('mode') == 'mask',
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
if preferences.get('navigator', {}).get('userAgent'):
|
|
292
|
-
preferences['userAgent'] = preferences.get('navigator', {}).get('userAgent')
|
|
293
|
-
|
|
294
|
-
if preferences.get('navigator', {}).get('doNotTrack'):
|
|
295
|
-
preferences['doNotTrack'] = preferences.get('navigator', {}).get('doNotTrack')
|
|
296
|
-
|
|
297
|
-
if preferences.get('navigator', {}).get('hardwareConcurrency'):
|
|
298
|
-
preferences['hardwareConcurrency'] = preferences.get('navigator', {}).get('hardwareConcurrency')
|
|
299
|
-
|
|
300
|
-
if preferences.get('navigator', {}).get('language'):
|
|
301
|
-
preferences['language'] = preferences.get('navigator', {}).get('language')
|
|
302
|
-
|
|
303
|
-
return preferences
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
def updatePreferences(self):
|
|
307
|
-
pref_file = os.path.join(self.profile_path, 'Default/Preferences')
|
|
308
|
-
with open(pref_file, 'r', encoding="utf-8") as pfile:
|
|
309
|
-
preferences = json.load(pfile)
|
|
310
|
-
profile = self.profile
|
|
311
|
-
proxy = self.profile.get('proxy')
|
|
312
|
-
# print('proxy=', proxy)
|
|
313
|
-
if proxy and (proxy.get('mode')=='gologin' or proxy.get('mode')=='tor'):
|
|
314
|
-
autoProxyServer = profile.get('autoProxyServer')
|
|
315
|
-
splittedAutoProxyServer = autoProxyServer.split('://')
|
|
316
|
-
splittedProxyAddress = splittedAutoProxyServer[1].split(':')
|
|
317
|
-
port = splittedProxyAddress[1]
|
|
318
|
-
|
|
319
|
-
proxy = {
|
|
320
|
-
'mode': 'http',
|
|
321
|
-
'host': splittedProxyAddress[0],
|
|
322
|
-
'port': port,
|
|
323
|
-
'username': profile.get('autoProxyUsername'),
|
|
324
|
-
'password': profile.get('autoProxyPassword'),
|
|
325
|
-
'timezone': profile.get('autoProxyTimezone', 'us'),
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
profile['proxy']['username'] = profile.get('autoProxyUsername')
|
|
329
|
-
profile['proxy']['password'] = profile.get('autoProxyPassword')
|
|
330
|
-
|
|
331
|
-
if not proxy or proxy.get('mode')=='none':
|
|
332
|
-
print('no proxy')
|
|
333
|
-
proxy = None
|
|
334
|
-
|
|
335
|
-
if proxy and proxy.get('mode')==None:
|
|
336
|
-
proxy['mode'] = 'http'
|
|
337
|
-
|
|
338
|
-
self.proxy = proxy
|
|
339
|
-
self.profile_name = profile.get('name')
|
|
340
|
-
if self.profile_name==None:
|
|
341
|
-
print('empty profile name')
|
|
342
|
-
print('profile=', profile)
|
|
343
|
-
exit()
|
|
344
|
-
|
|
345
|
-
gologin = self.convertPreferences(profile)
|
|
346
|
-
if self.credentials_enable_service!=None:
|
|
347
|
-
preferences['credentials_enable_service'] = self.credentials_enable_service
|
|
348
|
-
preferences['gologin'] = gologin
|
|
349
|
-
pfile = open(pref_file, 'w')
|
|
350
|
-
json.dump(preferences, pfile)
|
|
351
|
-
|
|
352
|
-
def createStartup(self):
|
|
353
|
-
if self.local==False and os.path.exists(self.profile_path):
|
|
354
|
-
shutil.rmtree(self.profile_path)
|
|
355
|
-
self.profile = self.getProfile()
|
|
356
|
-
if self.local==False:
|
|
357
|
-
self.downloadProfileZip()
|
|
358
|
-
self.updatePreferences()
|
|
359
|
-
return self.profile_path
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
def headers(self):
|
|
363
|
-
return {
|
|
364
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
365
|
-
'User-Agent': 'Selenium-API'
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
def getRandomFingerprint(self, options):
|
|
370
|
-
os_type = options.get('os', 'lin')
|
|
371
|
-
return json.loads(requests.get(API_URL + '/browser/fingerprint?os=' + os_type, headers=self.headers()).content.decode('utf-8'))
|
|
372
|
-
|
|
373
|
-
def profiles(self):
|
|
374
|
-
return json.loads(requests.get(API_URL + '/browser/v2', headers=self.headers()).content.decode('utf-8'))
|
|
375
|
-
|
|
376
|
-
def create(self, options={}):
|
|
377
|
-
profile_options = self.getRandomFingerprint(options)
|
|
378
|
-
profile = {
|
|
379
|
-
"name": "default_name",
|
|
380
|
-
"notes": "auto generated",
|
|
381
|
-
"browserType": "chrome",
|
|
382
|
-
"os": "lin",
|
|
383
|
-
"startUrl": "google.com",
|
|
384
|
-
"googleServicesEnabled": True,
|
|
385
|
-
"lockEnabled": False,
|
|
386
|
-
"audioContext": {
|
|
387
|
-
"mode": "noise"
|
|
388
|
-
},
|
|
389
|
-
"canvas": {
|
|
390
|
-
"mode": "noise"
|
|
391
|
-
},
|
|
392
|
-
"webRTC": {
|
|
393
|
-
"mode": "disabled",
|
|
394
|
-
"enabled": False,
|
|
395
|
-
"customize": True,
|
|
396
|
-
"fillBasedOnIp": True
|
|
397
|
-
},
|
|
398
|
-
"navigator": profile_options.get('navigator', {}),
|
|
399
|
-
"screenHeight": 768,
|
|
400
|
-
"screenWidth": 1024,
|
|
401
|
-
"proxyEnabled": True,
|
|
402
|
-
"profile": json.dumps(profile_options),
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
if profile.get('navigator'):
|
|
406
|
-
profile['navigator']['resolution'] = "1024x768"
|
|
407
|
-
else:
|
|
408
|
-
profile['navigator'] = {'resolution': "1024x768"}
|
|
409
|
-
|
|
410
|
-
for k,v in options.items():
|
|
411
|
-
profile[k] = v
|
|
412
|
-
|
|
413
|
-
response = json.loads(requests.post(API_URL + '/browser/', headers=self.headers(), json=profile).content.decode('utf-8'))
|
|
414
|
-
return response.get('id')
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
def delete(self, profile_id=None):
|
|
418
|
-
profile = self.profile_id if profile_id==None else profile_id
|
|
419
|
-
requests.delete(API_URL + '/browser/' + profile, headers=self.headers())
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
def update(self, options):
|
|
423
|
-
self.profile_id = options.get('id')
|
|
424
|
-
profile = self.getProfile()
|
|
425
|
-
#print("profile", profile)
|
|
426
|
-
for k,v in options.items():
|
|
427
|
-
profile[k] = v
|
|
428
|
-
resp = requests.put(API_URL + '/browser/' + self.profile_id, headers=self.headers(), json=profile).content.decode('utf-8')
|
|
429
|
-
#print("update", resp)
|
|
430
|
-
#return json.loads(resp)
|
|
431
|
-
|
|
432
|
-
def waitDebuggingUrl(self, delay_s, try_count=3):
|
|
433
|
-
url = 'https://' + self.profile_id + '.orbita.gologin.com/json/version'
|
|
434
|
-
wsUrl = ''
|
|
435
|
-
try_number = 1
|
|
436
|
-
while wsUrl=='':
|
|
437
|
-
time.sleep(delay_s)
|
|
438
|
-
try:
|
|
439
|
-
response = json.loads(requests.get(url).content)
|
|
440
|
-
wsUrl = response.get('webSocketDebuggerUrl', '')
|
|
441
|
-
except:
|
|
442
|
-
pass
|
|
443
|
-
if try_number >= try_count:
|
|
444
|
-
return {'status': 'failure', 'wsUrl': wsUrl}
|
|
445
|
-
try_number += 1
|
|
446
|
-
|
|
447
|
-
wsUrl = wsUrl.replace('ws://', 'wss://').replace('127.0.0.1', self.profile_id + '.orbita.gologin.com')
|
|
448
|
-
return {'status': 'success', 'wsUrl': wsUrl}
|
|
449
|
-
|
|
450
|
-
def startRemote(self, delay_s=3):
|
|
451
|
-
profileResponse = requests.post(API_URL + '/browser/' + self.profile_id + '/web', headers=self.headers()).content.decode('utf-8')
|
|
452
|
-
print('profileResponse', profileResponse)
|
|
453
|
-
if profileResponse == 'ok':
|
|
454
|
-
return self.waitDebuggingUrl(delay_s)
|
|
455
|
-
return {'status': 'failure'}
|
|
456
|
-
|
|
457
|
-
def stopRemote(self):
|
|
458
|
-
requests.delete(API_URL + '/browser/' + self.profile_id + '/web', headers=self.headers())
|
|
Binary file
|