gologin 1.0.33 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gologin.js +50 -29
- package/package.json +1 -2
- package/selenium/chromedriver +0 -0
- package/selenium/chromedriver.exe +0 -0
- package/selenium/example.js +0 -32
- package/selenium/gologin-create-profile.py +0 -30
- package/selenium/gologin-local.py +0 -30
- package/selenium/gologin-selenium-multiprocess.py +0 -50
- package/selenium/gologin-selenium.py +0 -28
- package/selenium/gologin.py +0 -458
- package/selenium/mac/chromedriver +0 -0
package/gologin.js
CHANGED
|
@@ -100,7 +100,7 @@ class GoLogin {
|
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
async profiles() {
|
|
103
|
-
const profilesResponse = await requests.get(`${API_URL}/browser
|
|
103
|
+
const profilesResponse = await requests.get(`${API_URL}/browser/v2`, {
|
|
104
104
|
headers: {
|
|
105
105
|
'Authorization': `Bearer ${this.access_token}`,
|
|
106
106
|
'User-Agent': 'gologin-api',
|
|
@@ -124,6 +124,11 @@ class GoLogin {
|
|
|
124
124
|
}
|
|
125
125
|
})
|
|
126
126
|
debug(profileResponse.body);
|
|
127
|
+
|
|
128
|
+
if (profileResponse.statusCode === 404) {
|
|
129
|
+
throw new Error(JSON.parse(profileResponse.body).message);
|
|
130
|
+
}
|
|
131
|
+
|
|
127
132
|
if (profileResponse.statusCode !== 200) {
|
|
128
133
|
throw new Error(`Gologin /browser/${id} response error ${profileResponse.statusCode} INVALID TOKEN OR PROFILE NOT FOUND`);
|
|
129
134
|
}
|
|
@@ -132,6 +137,7 @@ class GoLogin {
|
|
|
132
137
|
throw new Error("invalid token");
|
|
133
138
|
}
|
|
134
139
|
|
|
140
|
+
|
|
135
141
|
return JSON.parse(profileResponse.body);
|
|
136
142
|
}
|
|
137
143
|
|
|
@@ -450,6 +456,10 @@ class GoLogin {
|
|
|
450
456
|
await BrowserUserDataManager.composeFonts(families, profilePath, this.differentOs);
|
|
451
457
|
}
|
|
452
458
|
|
|
459
|
+
const [languages] = this.language.split(';');
|
|
460
|
+
preferences.gologin.langHeader = gologin.language;
|
|
461
|
+
preferences.gologin.languages = languages;
|
|
462
|
+
|
|
453
463
|
await writeFile(path.join(profilePath, 'Default', 'Preferences'), JSON.stringify(_.merge(preferences, {
|
|
454
464
|
gologin
|
|
455
465
|
})));
|
|
@@ -533,14 +543,22 @@ class GoLogin {
|
|
|
533
543
|
let data = null;
|
|
534
544
|
if (proxy!==null && proxy.mode !== "none") {
|
|
535
545
|
if (proxy.mode.includes('socks')) {
|
|
536
|
-
|
|
546
|
+
for(let i=0; i<5; i++){
|
|
547
|
+
try{
|
|
548
|
+
debug('getting timeZone socks try', i+1);
|
|
549
|
+
return this.getTimezoneWithSocks(proxy);
|
|
550
|
+
} catch (e) {
|
|
551
|
+
console.log(e.message);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
throw new Error(`Socks proxy connection timed out`);
|
|
537
555
|
}
|
|
538
556
|
|
|
539
557
|
const proxyUrl = `${proxy.mode}://${proxy.username}:${proxy.password}@${proxy.host}:${proxy.port}`;
|
|
540
|
-
debug('getTimeZone start https://time.gologin.com', proxyUrl);
|
|
541
|
-
data = await requests.get('https://time.gologin.com', { proxy: proxyUrl, timeout: 20 * 1000, maxAttempts: 5 });
|
|
558
|
+
debug('getTimeZone start https://time.gologin.com/timezone', proxyUrl);
|
|
559
|
+
data = await requests.get('https://time.gologin.com/timezone', { proxy: proxyUrl, timeout: 20 * 1000, maxAttempts: 5 });
|
|
542
560
|
} else {
|
|
543
|
-
data = await requests.get('https://time.gologin.com', { timeout: 20 * 1000, maxAttempts: 5 });
|
|
561
|
+
data = await requests.get('https://time.gologin.com/timezone', { timeout: 20 * 1000, maxAttempts: 5 });
|
|
544
562
|
}
|
|
545
563
|
debug('getTimeZone finish', data.body);
|
|
546
564
|
this._tz = JSON.parse(data.body);
|
|
@@ -1135,36 +1153,39 @@ class GoLogin {
|
|
|
1135
1153
|
}
|
|
1136
1154
|
|
|
1137
1155
|
debug('profileResponse', profileResponse.statusCode, profileResponse.body);
|
|
1156
|
+
/*
|
|
1138
1157
|
if (profileResponse.statusCode !== 202) {
|
|
1139
1158
|
return {'status': 'failure', 'code': profileResponse.statusCode};
|
|
1140
1159
|
}
|
|
1160
|
+
*/
|
|
1141
1161
|
|
|
1142
|
-
if (profileResponse.body === 'ok') {
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
const {
|
|
1155
|
-
resolution = '1920x1080',
|
|
1156
|
-
language = 'en-US,en;q=0.9',
|
|
1157
|
-
} = navigator;
|
|
1158
|
-
this.language = language;
|
|
1159
|
-
const [screenWidth, screenHeight] = resolution.split('x');
|
|
1160
|
-
this.resolution = {
|
|
1161
|
-
width: parseInt(screenWidth, 10),
|
|
1162
|
-
height: parseInt(screenHeight, 10),
|
|
1163
|
-
};
|
|
1162
|
+
// if (profileResponse.body === 'ok') {
|
|
1163
|
+
const profile = await this.getProfile();
|
|
1164
|
+
const { navigator = {}, fonts, os: profileOs } = profile;
|
|
1165
|
+
this.fontsMasking = fonts?.enableMasking;
|
|
1166
|
+
this.profileOs = profileOs;
|
|
1167
|
+
this.differentOs =
|
|
1168
|
+
profileOs !== 'android' && (
|
|
1169
|
+
OS_PLATFORM === 'win32' && profileOs !== 'win' ||
|
|
1170
|
+
OS_PLATFORM === 'darwin' && profileOs !== 'mac' ||
|
|
1171
|
+
OS_PLATFORM === 'linux' && profileOs !== 'lin'
|
|
1172
|
+
);
|
|
1164
1173
|
|
|
1165
|
-
|
|
1174
|
+
const {
|
|
1175
|
+
resolution = '1920x1080',
|
|
1176
|
+
language = 'en-US,en;q=0.9',
|
|
1177
|
+
} = navigator;
|
|
1178
|
+
this.language = language;
|
|
1179
|
+
const [screenWidth, screenHeight] = resolution.split('x');
|
|
1180
|
+
this.resolution = {
|
|
1181
|
+
width: parseInt(screenWidth, 10),
|
|
1182
|
+
height: parseInt(screenHeight, 10),
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
let wsUrl = await this.waitDebuggingUrl(delay_ms);
|
|
1186
|
+
if(wsUrl!=''){
|
|
1166
1187
|
return { 'status': 'success', wsUrl }
|
|
1167
|
-
}
|
|
1188
|
+
}
|
|
1168
1189
|
|
|
1169
1190
|
return { 'status': 'failure', 'message': profileResponse.body };
|
|
1170
1191
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gologin",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.36",
|
|
4
4
|
"description": "A high-level API to control Orbita browser over GoLogin API",
|
|
5
5
|
"main": "./gologin.js",
|
|
6
6
|
"repository": {
|
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
"dependencies": {
|
|
16
16
|
"archiver": "^3.1.1",
|
|
17
17
|
"child_process": "^1.0.2",
|
|
18
|
-
"chromedriver": "^89.0.0",
|
|
19
18
|
"decompress": "^4.2.1",
|
|
20
19
|
"decompress-unzip": "^4.0.1",
|
|
21
20
|
"form-data": "^3.0.0",
|
package/selenium/chromedriver
DELETED
|
Binary file
|
|
Binary file
|
package/selenium/example.js
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
const GoLogin = require('../gologin');
|
|
2
|
-
const webdriver = require("selenium-webdriver");
|
|
3
|
-
const chrome = require("selenium-webdriver/chrome");
|
|
4
|
-
chrome.setDefaultService(new chrome.ServiceBuilder('./chromedriver').build());
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
(async () =>{
|
|
8
|
-
const GL = new GoLogin({
|
|
9
|
-
token: 'yU0token',
|
|
10
|
-
profile_id: 'yU0Pr0f1leiD',
|
|
11
|
-
executablePath: '/usr/bin/orbita-browser/chrome',
|
|
12
|
-
});
|
|
13
|
-
console.log('creating startup')
|
|
14
|
-
await GL.createStartup();
|
|
15
|
-
console.log('spawn arguments')
|
|
16
|
-
const arguments = await GL.spawnArguments();
|
|
17
|
-
console.log('set options')
|
|
18
|
-
const chromeOptions = new chrome.Options();
|
|
19
|
-
arguments.forEach((e) => {
|
|
20
|
-
console.log('e=', e);
|
|
21
|
-
chromeOptions.addArguments(e)
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
chromeOptions.setChromeBinaryPath('/usr/bin/orbita-browser/chrome');
|
|
25
|
-
|
|
26
|
-
driver = new webdriver.Builder()
|
|
27
|
-
.forBrowser("chrome")
|
|
28
|
-
.setChromeOptions(chromeOptions)
|
|
29
|
-
.build();
|
|
30
|
-
|
|
31
|
-
await driver.get('https://myip.link')
|
|
32
|
-
})();
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
from gologin import GoLogin
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
gl = GoLogin({
|
|
5
|
-
"token": "yU0token",
|
|
6
|
-
})
|
|
7
|
-
|
|
8
|
-
profile_id = gl.create({
|
|
9
|
-
"name": 'profile_mac',
|
|
10
|
-
"os": 'mac',
|
|
11
|
-
"navigator": {
|
|
12
|
-
"language": 'enUS',
|
|
13
|
-
"userAgent": 'MyUserAgent',
|
|
14
|
-
"resolution": '1024x768',
|
|
15
|
-
"platform": 'mac',
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
print('profile id=', profile_id);
|
|
20
|
-
|
|
21
|
-
gl.update({
|
|
22
|
-
"id": profile_id,
|
|
23
|
-
"name": 'profile_mac2',
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
profile = gl.getProfile(profile_id);
|
|
27
|
-
|
|
28
|
-
print('new profile name=', profile.get("name"));
|
|
29
|
-
|
|
30
|
-
gl.delete(profile_id)
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
from sys import platform
|
|
3
|
-
from selenium import webdriver
|
|
4
|
-
from selenium.webdriver.chrome.options import Options
|
|
5
|
-
from gologin import GoLogin
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
gl = GoLogin({
|
|
9
|
-
"token": "yU0token",
|
|
10
|
-
"profile_id": "yU0Pr0f1leiD",
|
|
11
|
-
"local": True,
|
|
12
|
-
"credentials_enable_service": False,
|
|
13
|
-
})
|
|
14
|
-
|
|
15
|
-
if platform == "linux" or platform == "linux2":
|
|
16
|
-
chrome_driver_path = "./chromedriver"
|
|
17
|
-
elif platform == "darwin":
|
|
18
|
-
chrome_driver_path = "./mac/chromedriver"
|
|
19
|
-
elif platform == "win32":
|
|
20
|
-
chrome_driver_path = "chromedriver.exe"
|
|
21
|
-
|
|
22
|
-
debugger_address = gl.start()
|
|
23
|
-
chrome_options = Options()
|
|
24
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
25
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
26
|
-
driver.get("http://www.python.org")
|
|
27
|
-
assert "Python" in driver.title
|
|
28
|
-
driver.close()
|
|
29
|
-
time.sleep(3)
|
|
30
|
-
gl.stop()
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
import os
|
|
3
|
-
from multiprocessing import Pool
|
|
4
|
-
from sys import platform
|
|
5
|
-
from selenium import webdriver
|
|
6
|
-
from selenium.webdriver.chrome.options import Options
|
|
7
|
-
from gologin import GoLogin
|
|
8
|
-
|
|
9
|
-
def scrap(profile):
|
|
10
|
-
gl = GoLogin({
|
|
11
|
-
'token': 'yU0token',
|
|
12
|
-
'profile_id': profile['profile_id'],
|
|
13
|
-
'port': profile['port'],
|
|
14
|
-
})
|
|
15
|
-
|
|
16
|
-
if platform == "linux" or platform == "linux2":
|
|
17
|
-
chrome_driver_path = './chromedriver'
|
|
18
|
-
elif platform == "darwin":
|
|
19
|
-
chrome_driver_path = './mac/chromedriver'
|
|
20
|
-
elif platform == "win32":
|
|
21
|
-
chrome_driver_path = 'chromedriver.exe'
|
|
22
|
-
|
|
23
|
-
debugger_address = gl.start()
|
|
24
|
-
chrome_options = Options()
|
|
25
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
26
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
27
|
-
driver.get("http://www.python.org")
|
|
28
|
-
print('ready', profile['profile_id'], driver.title)
|
|
29
|
-
time.sleep(10)
|
|
30
|
-
print('closing', profile['profile_id'])
|
|
31
|
-
driver.close()
|
|
32
|
-
gl.stop()
|
|
33
|
-
|
|
34
|
-
profiles = [
|
|
35
|
-
{'profile_id': 'profile_id_1', 'port': 3500},
|
|
36
|
-
{'profile_id': 'profile_id_2', 'port': 3501},
|
|
37
|
-
{'profile_id': 'profile_id_3', 'port': 3502},
|
|
38
|
-
]
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
with Pool(3) as p:
|
|
42
|
-
p.map(scrap, profiles)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
if platform == "win32":
|
|
46
|
-
os.system('taskkill /im chrome.exe /f')
|
|
47
|
-
os.system('taskkill /im chromedriver.exe /f')
|
|
48
|
-
else:
|
|
49
|
-
os.system('killall -9 chrome')
|
|
50
|
-
os.system('killall -9 chromedriver')
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
from sys import platform
|
|
3
|
-
from selenium import webdriver
|
|
4
|
-
from selenium.webdriver.chrome.options import Options
|
|
5
|
-
from gologin import GoLogin
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
gl = GoLogin({
|
|
9
|
-
"token": "yU0token",
|
|
10
|
-
"profile_id": "yU0Pr0f1leiD",
|
|
11
|
-
})
|
|
12
|
-
|
|
13
|
-
if platform == "linux" or platform == "linux2":
|
|
14
|
-
chrome_driver_path = "./chromedriver"
|
|
15
|
-
elif platform == "darwin":
|
|
16
|
-
chrome_driver_path = "./mac/chromedriver"
|
|
17
|
-
elif platform == "win32":
|
|
18
|
-
chrome_driver_path = "chromedriver.exe"
|
|
19
|
-
|
|
20
|
-
debugger_address = gl.start()
|
|
21
|
-
chrome_options = Options()
|
|
22
|
-
chrome_options.add_experimental_option("debuggerAddress", debugger_address)
|
|
23
|
-
driver = webdriver.Chrome(executable_path=chrome_driver_path, options=chrome_options)
|
|
24
|
-
driver.get("http://www.python.org")
|
|
25
|
-
assert "Python" in driver.title
|
|
26
|
-
driver.close()
|
|
27
|
-
time.sleep(3)
|
|
28
|
-
gl.stop()
|
package/selenium/gologin.py
DELETED
|
@@ -1,458 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
import time
|
|
3
|
-
import os
|
|
4
|
-
import stat
|
|
5
|
-
import sys
|
|
6
|
-
import shutil
|
|
7
|
-
import requests
|
|
8
|
-
import zipfile
|
|
9
|
-
import subprocess
|
|
10
|
-
import pathlib
|
|
11
|
-
import tempfile
|
|
12
|
-
|
|
13
|
-
API_URL = 'https://api.gologin.com'
|
|
14
|
-
|
|
15
|
-
class GoLogin(object):
|
|
16
|
-
def __init__(self, options):
|
|
17
|
-
self.access_token = options.get('token')
|
|
18
|
-
|
|
19
|
-
self.tmpdir = options.get('tmpdir', tempfile.gettempdir())
|
|
20
|
-
self.address = options.get('address', '127.0.0.1')
|
|
21
|
-
self.extra_params = options.get('extra_params', [])
|
|
22
|
-
self.port = options.get('port', 3500)
|
|
23
|
-
self.local = options.get('local', False)
|
|
24
|
-
self.spawn_browser = options.get('spawn_browser', True)
|
|
25
|
-
self.credentials_enable_service = options.get('credentials_enable_service')
|
|
26
|
-
|
|
27
|
-
home = str(pathlib.Path.home())
|
|
28
|
-
self.executablePath = options.get('executablePath', os.path.join(home, '.gologin/browser/orbita-browser/chrome'))
|
|
29
|
-
print('executablePath', self.executablePath)
|
|
30
|
-
if self.extra_params:
|
|
31
|
-
print('extra_params', self.extra_params)
|
|
32
|
-
self.setProfileId(options.get('profile_id'))
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def setProfileId(self, profile_id):
|
|
36
|
-
self.profile_id = profile_id
|
|
37
|
-
if self.profile_id==None:
|
|
38
|
-
return
|
|
39
|
-
self.profile_path = os.path.join(self.tmpdir, 'gologin_'+self.profile_id)
|
|
40
|
-
self.profile_zip_path = os.path.join(self.tmpdir, 'gologin_'+self.profile_id+'.zip')
|
|
41
|
-
self.profile_zip_path_upload = os.path.join(self.tmpdir, 'gologin_'+self.profile_id+'_upload.zip')
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def spawnBrowser(self):
|
|
45
|
-
proxy = self.proxy
|
|
46
|
-
proxy_host = ''
|
|
47
|
-
if proxy:
|
|
48
|
-
if proxy.get('mode')==None or proxy.get('mode')=='geolocation':
|
|
49
|
-
proxy['mode'] = 'http'
|
|
50
|
-
proxy_host = proxy.get('host')
|
|
51
|
-
proxy = self.formatProxyUrl(proxy)
|
|
52
|
-
|
|
53
|
-
tz = self.tz.get('timezone')
|
|
54
|
-
|
|
55
|
-
params = [
|
|
56
|
-
self.executablePath,
|
|
57
|
-
'--remote-debugging-port='+str(self.port),
|
|
58
|
-
'--user-data-dir='+self.profile_path,
|
|
59
|
-
'--password-store=basic',
|
|
60
|
-
'--tz='+tz,
|
|
61
|
-
'--gologin-profile='+self.profile_name,
|
|
62
|
-
'--lang=en',
|
|
63
|
-
]
|
|
64
|
-
if proxy:
|
|
65
|
-
hr_rules = '"MAP * 0.0.0.0 , EXCLUDE %s"'%(proxy_host)
|
|
66
|
-
params.append('--proxy-server='+proxy)
|
|
67
|
-
params.append('--host-resolver-rules='+hr_rules)
|
|
68
|
-
|
|
69
|
-
for param in self.extra_params:
|
|
70
|
-
params.append(param)
|
|
71
|
-
|
|
72
|
-
if sys.platform == "darwin":
|
|
73
|
-
subprocess.Popen(params)
|
|
74
|
-
else:
|
|
75
|
-
subprocess.Popen(params, start_new_session=True)
|
|
76
|
-
|
|
77
|
-
try_count = 1
|
|
78
|
-
url = str(self.address) + ':' + str(self.port)
|
|
79
|
-
while try_count<100:
|
|
80
|
-
try:
|
|
81
|
-
data = requests.get('http://'+url+'/json').content
|
|
82
|
-
break
|
|
83
|
-
except:
|
|
84
|
-
try_count += 1
|
|
85
|
-
time.sleep(1)
|
|
86
|
-
|
|
87
|
-
return url
|
|
88
|
-
|
|
89
|
-
def start(self):
|
|
90
|
-
profile_path = self.createStartup()
|
|
91
|
-
if self.spawn_browser == True:
|
|
92
|
-
return self.spawnBrowser()
|
|
93
|
-
return profile_path
|
|
94
|
-
|
|
95
|
-
def zipdir(self, path, ziph):
|
|
96
|
-
for root, dirs, files in os.walk(path):
|
|
97
|
-
for file in files:
|
|
98
|
-
path = os.path.join(root, file)
|
|
99
|
-
if not os.path.exists(path):
|
|
100
|
-
continue
|
|
101
|
-
if stat.S_ISSOCK(os.stat(path).st_mode):
|
|
102
|
-
continue
|
|
103
|
-
try:
|
|
104
|
-
ziph.write(path, path.replace(self.profile_path, ''))
|
|
105
|
-
except:
|
|
106
|
-
continue
|
|
107
|
-
|
|
108
|
-
def stop(self):
|
|
109
|
-
self.sanitizeProfile()
|
|
110
|
-
if self.local==False:
|
|
111
|
-
self.commitProfile()
|
|
112
|
-
os.remove(self.profile_zip_path_upload)
|
|
113
|
-
shutil.rmtree(self.profile_path)
|
|
114
|
-
|
|
115
|
-
def commitProfile(self):
|
|
116
|
-
zipf = zipfile.ZipFile(self.profile_zip_path_upload, 'w', zipfile.ZIP_DEFLATED)
|
|
117
|
-
self.zipdir(self.profile_path, zipf)
|
|
118
|
-
zipf.close()
|
|
119
|
-
|
|
120
|
-
headers = {
|
|
121
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
122
|
-
'User-Agent': 'Selenium-API'
|
|
123
|
-
}
|
|
124
|
-
# print('profile size=', os.stat(self.profile_zip_path_upload).st_size)
|
|
125
|
-
|
|
126
|
-
signedUrl = requests.get(API_URL + '/browser/' + self.profile_id + '/storage-signature', headers=headers).content.decode('utf-8')
|
|
127
|
-
|
|
128
|
-
requests.put(signedUrl, data=open(self.profile_zip_path_upload, 'rb'))
|
|
129
|
-
|
|
130
|
-
# print('commit profile complete')
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def sanitizeProfile(self):
|
|
134
|
-
remove_dirs = [
|
|
135
|
-
'Default/Cache',
|
|
136
|
-
'Default/Service Worker/CacheStorage',
|
|
137
|
-
'Default/Code Cache',
|
|
138
|
-
'Default/GPUCache',
|
|
139
|
-
'GrShaderCache',
|
|
140
|
-
'ShaderCache',
|
|
141
|
-
'biahpgbdmdkfgndcmfiipgcebobojjkp',
|
|
142
|
-
'afalakplffnnnlkncjhbmahjfjhmlkal',
|
|
143
|
-
'cffkpbalmllkdoenhmdmpbkajipdjfam',
|
|
144
|
-
'Dictionaries',
|
|
145
|
-
'enkheaiicpeffbfgjiklngbpkilnbkoi',
|
|
146
|
-
'oofiananboodjbbmdelgdommihjbkfag',
|
|
147
|
-
'SafetyTips',
|
|
148
|
-
'fonts',
|
|
149
|
-
];
|
|
150
|
-
|
|
151
|
-
for d in remove_dirs:
|
|
152
|
-
fpath = os.path.join(self.profile_path, d)
|
|
153
|
-
if os.path.exists(fpath):
|
|
154
|
-
try:
|
|
155
|
-
shutil.rmtree(fpath)
|
|
156
|
-
except:
|
|
157
|
-
continue
|
|
158
|
-
|
|
159
|
-
def formatProxyUrl(self, proxy):
|
|
160
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
161
|
-
|
|
162
|
-
def formatProxyUrlPassword(self, proxy):
|
|
163
|
-
if proxy.get('username', '')=='':
|
|
164
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
165
|
-
else:
|
|
166
|
-
return proxy.get('mode', 'http')+'://'+proxy.get('username','')+':'+proxy.get('password')+'@'+proxy.get('host','')+':'+str(proxy.get('port',80))
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
def getTimeZone(self):
|
|
170
|
-
proxy = self.proxy
|
|
171
|
-
if proxy:
|
|
172
|
-
proxies = {proxy.get('mode'): self.formatProxyUrlPassword(proxy)}
|
|
173
|
-
data = requests.get('https://time.gologin.com', proxies=proxies)
|
|
174
|
-
else:
|
|
175
|
-
data = requests.get('https://time.gologin.com')
|
|
176
|
-
return json.loads(data.content.decode('utf-8'))
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
def getProfile(self, profile_id=None):
|
|
180
|
-
profile = self.profile_id if profile_id==None else profile_id
|
|
181
|
-
headers = {
|
|
182
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
183
|
-
'User-Agent': 'Selenium-API'
|
|
184
|
-
}
|
|
185
|
-
return json.loads(requests.get(API_URL + '/browser/' + profile, headers=headers).content.decode('utf-8'))
|
|
186
|
-
|
|
187
|
-
def downloadProfileZip(self):
|
|
188
|
-
s3path = self.profile.get('s3Path', '')
|
|
189
|
-
data = ''
|
|
190
|
-
if s3path=='':
|
|
191
|
-
# print('downloading profile direct')
|
|
192
|
-
headers = {
|
|
193
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
194
|
-
'User-Agent': 'Selenium-API'
|
|
195
|
-
}
|
|
196
|
-
data = requests.get(API_URL + '/browser/'+self.profile_id, headers=headers).content
|
|
197
|
-
else:
|
|
198
|
-
# print('downloading profile s3')
|
|
199
|
-
s3url = 'https://gprofiles.gologin.com/' + s3path.replace(' ', '+')
|
|
200
|
-
data = requests.get(s3url).content
|
|
201
|
-
|
|
202
|
-
if len(data)==0:
|
|
203
|
-
self.createEmptyProfile()
|
|
204
|
-
else:
|
|
205
|
-
with open(self.profile_zip_path, 'wb') as f:
|
|
206
|
-
f.write(data)
|
|
207
|
-
|
|
208
|
-
try:
|
|
209
|
-
self.extractProfileZip()
|
|
210
|
-
except:
|
|
211
|
-
self.createEmptyProfile()
|
|
212
|
-
self.extractProfileZip()
|
|
213
|
-
|
|
214
|
-
if not os.path.exists(os.path.join(self.profile_path, 'Default/Preferences')):
|
|
215
|
-
self.createEmptyProfile()
|
|
216
|
-
self.extractProfileZip()
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def createEmptyProfile(self):
|
|
220
|
-
print('createEmptyProfile')
|
|
221
|
-
empty_profile = '../gologin_zeroprofile.zip'
|
|
222
|
-
if not os.path.exists(empty_profile):
|
|
223
|
-
empty_profile = 'gologin_zeroprofile.zip'
|
|
224
|
-
shutil.copy(empty_profile, self.profile_zip_path)
|
|
225
|
-
|
|
226
|
-
def extractProfileZip(self):
|
|
227
|
-
with zipfile.ZipFile(self.profile_zip_path, 'r') as zip_ref:
|
|
228
|
-
zip_ref.extractall(self.profile_path)
|
|
229
|
-
os.remove(self.profile_zip_path)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
def getGeolocationParams(self, profileGeolocationParams, tzGeolocationParams):
|
|
233
|
-
if profileGeolocationParams.get('fillBasedOnIp'):
|
|
234
|
-
return {
|
|
235
|
-
'mode': profileGeolocationParams['mode'],
|
|
236
|
-
'latitude': float(tzGeolocationParams['latitude']),
|
|
237
|
-
'longitude': float(tzGeolocationParams['longitude']),
|
|
238
|
-
'accuracy': float(tzGeolocationParams['accuracy']),
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
return {
|
|
242
|
-
'mode': profileGeolocationParams['mode'],
|
|
243
|
-
'latitude': profileGeolocationParams['latitude'],
|
|
244
|
-
'longitude': profileGeolocationParams['longitude'],
|
|
245
|
-
'accuracy': profileGeolocationParams['accuracy'],
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def convertPreferences(self, preferences):
|
|
250
|
-
resolution = preferences.get('resolution', '1920x1080')
|
|
251
|
-
preferences['screenWidth'] = int(resolution.split('x')[0])
|
|
252
|
-
preferences['screenHeight'] = int(resolution.split('x')[1])
|
|
253
|
-
|
|
254
|
-
self.tz = self.getTimeZone()
|
|
255
|
-
# print('tz=', self.tz)
|
|
256
|
-
tzGeoLocation = {
|
|
257
|
-
'latitude': self.tz.get('ll', [0, 0])[0],
|
|
258
|
-
'longitude': self.tz.get('ll', [0, 0])[1],
|
|
259
|
-
'accuracy': self.tz.get('accuracy', 0),
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
preferences['geoLocation'] = self.getGeolocationParams(preferences['geolocation'], tzGeoLocation)
|
|
263
|
-
|
|
264
|
-
preferences['webRtc'] = {
|
|
265
|
-
'mode': 'public' if preferences.get('webRTC',{}).get('mode') == 'alerted' else preferences.get('webRTC',{}).get('mode'),
|
|
266
|
-
'publicIP': self.tz['ip'] if preferences.get('webRTC',{}).get('fillBasedOnIp') else preferences.get('webRTC',{}).get('publicIp'),
|
|
267
|
-
'localIps': preferences.get('webRTC',{}).get('localIps', [])
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
preferences['timezone'] = {
|
|
271
|
-
'id': self.tz.get('timezone')
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
preferences['webgl_noise_value'] = preferences.get('webGL', {}).get('noise')
|
|
275
|
-
preferences['get_client_rects_noise'] = preferences.get('webGL', {}).get('getClientRectsNoise')
|
|
276
|
-
preferences['canvasMode'] = preferences.get('canvas', {}).get('mode')
|
|
277
|
-
preferences['canvasNoise'] = preferences.get('canvas', {}).get('noise')
|
|
278
|
-
preferences['audioContext'] = {
|
|
279
|
-
'enable': preferences.get('audioContext').get('mode', 'off'),
|
|
280
|
-
'noiseValue': preferences.get('audioContext').get('noise'),
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
preferences['webgl'] = {
|
|
284
|
-
'metadata': {
|
|
285
|
-
'vendor': preferences.get('webGLMetadata', {}).get('vendor'),
|
|
286
|
-
'renderer': preferences.get('webGLMetadata', {}).get('renderer'),
|
|
287
|
-
'mode': preferences.get('webGLMetadata', {}).get('mode') == 'mask',
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
if preferences.get('navigator', {}).get('userAgent'):
|
|
292
|
-
preferences['userAgent'] = preferences.get('navigator', {}).get('userAgent')
|
|
293
|
-
|
|
294
|
-
if preferences.get('navigator', {}).get('doNotTrack'):
|
|
295
|
-
preferences['doNotTrack'] = preferences.get('navigator', {}).get('doNotTrack')
|
|
296
|
-
|
|
297
|
-
if preferences.get('navigator', {}).get('hardwareConcurrency'):
|
|
298
|
-
preferences['hardwareConcurrency'] = preferences.get('navigator', {}).get('hardwareConcurrency')
|
|
299
|
-
|
|
300
|
-
if preferences.get('navigator', {}).get('language'):
|
|
301
|
-
preferences['language'] = preferences.get('navigator', {}).get('language')
|
|
302
|
-
|
|
303
|
-
return preferences
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
def updatePreferences(self):
|
|
307
|
-
pref_file = os.path.join(self.profile_path, 'Default/Preferences')
|
|
308
|
-
with open(pref_file, 'r', encoding="utf-8") as pfile:
|
|
309
|
-
preferences = json.load(pfile)
|
|
310
|
-
profile = self.profile
|
|
311
|
-
proxy = self.profile.get('proxy')
|
|
312
|
-
# print('proxy=', proxy)
|
|
313
|
-
if proxy and (proxy.get('mode')=='gologin' or proxy.get('mode')=='tor'):
|
|
314
|
-
autoProxyServer = profile.get('autoProxyServer')
|
|
315
|
-
splittedAutoProxyServer = autoProxyServer.split('://')
|
|
316
|
-
splittedProxyAddress = splittedAutoProxyServer[1].split(':')
|
|
317
|
-
port = splittedProxyAddress[1]
|
|
318
|
-
|
|
319
|
-
proxy = {
|
|
320
|
-
'mode': 'http',
|
|
321
|
-
'host': splittedProxyAddress[0],
|
|
322
|
-
'port': port,
|
|
323
|
-
'username': profile.get('autoProxyUsername'),
|
|
324
|
-
'password': profile.get('autoProxyPassword'),
|
|
325
|
-
'timezone': profile.get('autoProxyTimezone', 'us'),
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
profile['proxy']['username'] = profile.get('autoProxyUsername')
|
|
329
|
-
profile['proxy']['password'] = profile.get('autoProxyPassword')
|
|
330
|
-
|
|
331
|
-
if not proxy or proxy.get('mode')=='none':
|
|
332
|
-
print('no proxy')
|
|
333
|
-
proxy = None
|
|
334
|
-
|
|
335
|
-
if proxy and proxy.get('mode')==None:
|
|
336
|
-
proxy['mode'] = 'http'
|
|
337
|
-
|
|
338
|
-
self.proxy = proxy
|
|
339
|
-
self.profile_name = profile.get('name')
|
|
340
|
-
if self.profile_name==None:
|
|
341
|
-
print('empty profile name')
|
|
342
|
-
print('profile=', profile)
|
|
343
|
-
exit()
|
|
344
|
-
|
|
345
|
-
gologin = self.convertPreferences(profile)
|
|
346
|
-
if self.credentials_enable_service!=None:
|
|
347
|
-
preferences['credentials_enable_service'] = self.credentials_enable_service
|
|
348
|
-
preferences['gologin'] = gologin
|
|
349
|
-
pfile = open(pref_file, 'w')
|
|
350
|
-
json.dump(preferences, pfile)
|
|
351
|
-
|
|
352
|
-
def createStartup(self):
|
|
353
|
-
if self.local==False and os.path.exists(self.profile_path):
|
|
354
|
-
shutil.rmtree(self.profile_path)
|
|
355
|
-
self.profile = self.getProfile()
|
|
356
|
-
if self.local==False:
|
|
357
|
-
self.downloadProfileZip()
|
|
358
|
-
self.updatePreferences()
|
|
359
|
-
return self.profile_path
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
def headers(self):
|
|
363
|
-
return {
|
|
364
|
-
'Authorization': 'Bearer ' + self.access_token,
|
|
365
|
-
'User-Agent': 'Selenium-API'
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
def getRandomFingerprint(self, options):
|
|
370
|
-
os_type = options.get('os', 'lin')
|
|
371
|
-
return json.loads(requests.get(API_URL + '/browser/fingerprint?os=' + os_type, headers=self.headers()).content.decode('utf-8'))
|
|
372
|
-
|
|
373
|
-
def profiles(self):
|
|
374
|
-
return json.loads(requests.get(API_URL + '/browser/', headers=self.headers()).content.decode('utf-8'))
|
|
375
|
-
|
|
376
|
-
def create(self, options={}):
|
|
377
|
-
profile_options = self.getRandomFingerprint(options)
|
|
378
|
-
profile = {
|
|
379
|
-
"name": "default_name",
|
|
380
|
-
"notes": "auto generated",
|
|
381
|
-
"browserType": "chrome",
|
|
382
|
-
"os": "lin",
|
|
383
|
-
"startUrl": "google.com",
|
|
384
|
-
"googleServicesEnabled": True,
|
|
385
|
-
"lockEnabled": False,
|
|
386
|
-
"audioContext": {
|
|
387
|
-
"mode": "noise"
|
|
388
|
-
},
|
|
389
|
-
"canvas": {
|
|
390
|
-
"mode": "noise"
|
|
391
|
-
},
|
|
392
|
-
"webRTC": {
|
|
393
|
-
"mode": "disabled",
|
|
394
|
-
"enabled": False,
|
|
395
|
-
"customize": True,
|
|
396
|
-
"fillBasedOnIp": True
|
|
397
|
-
},
|
|
398
|
-
"navigator": profile_options.get('navigator', {}),
|
|
399
|
-
"screenHeight": 768,
|
|
400
|
-
"screenWidth": 1024,
|
|
401
|
-
"proxyEnabled": True,
|
|
402
|
-
"profile": json.dumps(profile_options),
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
if profile.get('navigator'):
|
|
406
|
-
profile['navigator']['resolution'] = "1024x768"
|
|
407
|
-
else:
|
|
408
|
-
profile['navigator'] = {'resolution': "1024x768"}
|
|
409
|
-
|
|
410
|
-
for k,v in options.items():
|
|
411
|
-
profile[k] = v
|
|
412
|
-
|
|
413
|
-
response = json.loads(requests.post(API_URL + '/browser/', headers=self.headers(), json=profile).content.decode('utf-8'))
|
|
414
|
-
return response.get('id')
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
def delete(self, profile_id=None):
|
|
418
|
-
profile = self.profile_id if profile_id==None else profile_id
|
|
419
|
-
requests.delete(API_URL + '/browser/' + profile, headers=self.headers())
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
def update(self, options):
|
|
423
|
-
self.profile_id = options.get('id')
|
|
424
|
-
profile = self.getProfile()
|
|
425
|
-
#print("profile", profile)
|
|
426
|
-
for k,v in options.items():
|
|
427
|
-
profile[k] = v
|
|
428
|
-
resp = requests.put(API_URL + '/browser/' + self.profile_id, headers=self.headers(), json=profile).content.decode('utf-8')
|
|
429
|
-
#print("update", resp)
|
|
430
|
-
#return json.loads(resp)
|
|
431
|
-
|
|
432
|
-
def waitDebuggingUrl(self, delay_s, try_count=3):
|
|
433
|
-
url = 'https://' + self.profile_id + '.orbita.gologin.com/json/version'
|
|
434
|
-
wsUrl = ''
|
|
435
|
-
try_number = 1
|
|
436
|
-
while wsUrl=='':
|
|
437
|
-
time.sleep(delay_s)
|
|
438
|
-
try:
|
|
439
|
-
response = json.loads(requests.get(url).content)
|
|
440
|
-
wsUrl = response.get('webSocketDebuggerUrl', '')
|
|
441
|
-
except:
|
|
442
|
-
pass
|
|
443
|
-
if try_number >= try_count:
|
|
444
|
-
return {'status': 'failure', 'wsUrl': wsUrl}
|
|
445
|
-
try_number += 1
|
|
446
|
-
|
|
447
|
-
wsUrl = wsUrl.replace('ws://', 'wss://').replace('127.0.0.1', self.profile_id + '.orbita.gologin.com')
|
|
448
|
-
return {'status': 'success', 'wsUrl': wsUrl}
|
|
449
|
-
|
|
450
|
-
def startRemote(self, delay_s=3):
|
|
451
|
-
profileResponse = requests.post(API_URL + '/browser/' + self.profile_id + '/web', headers=self.headers()).content.decode('utf-8')
|
|
452
|
-
print('profileResponse', profileResponse)
|
|
453
|
-
if profileResponse == 'ok':
|
|
454
|
-
return self.waitDebuggingUrl(delay_s)
|
|
455
|
-
return {'status': 'failure'}
|
|
456
|
-
|
|
457
|
-
def stopRemote(self):
|
|
458
|
-
requests.delete(API_URL + '/browser/' + self.profile_id + '/web', headers=self.headers())
|
|
Binary file
|