nodejs_chromium 1.0.8 → 1.1.1

package/index.js

@@ -6,32 +6,32 @@ global.__UA__ = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (K
 async function newChrome(params) {
 	let {
 		id = 'myChrome',
-			visible = false,
-			width = 1024,
-			height = 768,
-			scale = 1,
-			mobile = false,
-			dumpio = false,
-			debug = false,
-			incognito = true,
-			path = void 0,
-			slowMo = 1,
-			ua = void 0,
-			proxy = null,
+			visible = false, //是否打开浏览器
+			width = 1024, //宽
+			height = 768, //高
+			scale = 1, //缩放比例
+			mobile = false, //手机版
+			dumpio = false, //转发标准输入/输出流
+			devtools = false, //打开devtools
+			cache = false, //禁用缓存
+			incognito = true, //使用无痕模式启动
+			path = void 0, //Chrome路径，linux下必填
+			slowMo = 1, //每一步停留时间
+			ua = void 0, //若不指定ua则用__UA__
+			proxy = null, //代理，如：https://proxy.com:789/
 			cookies = null, //若cookies=false，则不处理cookies，不指定则由chrome处理，若=文件路径
-			abort_img = false,
-			no_cache = true,
+			abort = [], //要禁止的类型，常见["font","image","ping","stylesheet","document","fetch","script","xhr"]
 			headers = {},
 	} = params;
 
 	let option = {
-		userDataDir: `runtime/template/${id}`,
+		userDataDir: `runtime/.cache/${id}`,
 		// timeout: 500,//最大允许超时间，默认为3000，最小215，一般不要设置
 		slowMo, //每一步停留时间，不能太大，否则会太慢，特别是在类似写入很多Cookies时，每写入一个都要等一下。
 		headless: visible ? false : 'new', //'new',设置是否在无头模式下运行浏览器,false=会启动浏览器，true=无界面
-		devtools: !!debug, //打开调试
+		devtools: !!devtools, //打开调试
 		ignoreHTTPSErrors: true, //忽略 HTTPS 错误。屏蔽跳转不同域名的报错
-		dumpio: !!dumpio, //是否将浏览器的标准输入/输出流（stdio）的内容输出到 Node.js 的 stdout 和 stderr。
+		dumpio, //是否将浏览器的标准输入/输出流（stdio）的内容输出到 Node.js 的 stdout 和 stderr。
 		defaultViewport: {
 			width,
 			height,
@@ -52,8 +52,9 @@ async function newChrome(params) {
 			'--disable-web-security', //禁用浏览器的同源策略（Same-Origin Policy）和跨站请求伪造（CSRF）保护
 			`--window-size=${width},${height}`,
 			'--no-sandbox', //禁用沙箱模式
+			'--disable-autofill-backend', //禁止自动填充
 			'--disable-setuid-sandbox', //禁用 setuid 沙箱。这是另一种沙箱模式，通常用于 Linux 系统上。
-			// '--disable-infobars', //禁用 Chrome 在自动化控制时显示的信息栏。这个信息栏通常会告诉用户浏览器正在被自动化工具控制。
+			// '--disable-infobars', //禁用自动化控制时显示的信息栏，告诉用户浏览器正在被自动化工具控制。
 			'--disable-gpu', //禁用 GPU 加速。这通常用于在服务器环境或某些不支持 GPU 加速的平台上运行 Chrome。
 			'--disable-blink-features=AutomationControlled', //防止检测 blink 引擎特性判断是否处于自动化控制状态。
 			'--lang=zh-CN', //设置中文环境
@@ -64,6 +65,7 @@ async function newChrome(params) {
 
 	if (proxy) option.defaultArgs.push(`--proxy-server=${proxy}`);
 	if (incognito) option.args.push('--incognito', '--disable-infobars'); //使用无痕模式启动
+	if (!cache) option.args.push('--disable-cache'); //禁缓存
 	if (path) option.executablePath = path; //指定chrome安装路径
 
 	const browser = await puppeteer.launch(option);
@@ -77,8 +79,8 @@ async function newChrome(params) {
 		navigator.__proto__ = newProto;
 	});
 
-	if (cookies !== false) cookies = `runtime/cookies/${id}`;
-	const pageOption = { cookies, visible, abort_img, no_cache, headers };
+	if (cookies !== false) cookies = `runtime/.cache/${id}/cookies.json`;
+	const pageOption = { cookies, visible, abort, headers };
 	return new chrome(browser, page, pageOption);
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodejs_chromium",
-  "version": "1.0.8",
+  "version": "1.1.1",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/chrome.js

@@ -48,6 +48,7 @@ module.exports = class {
 	async clone() {
 		try {
 			const page = await this.browser.newPage();
+			await page.setBypassCSP(true); //绕过页面的内容安全策略
 			await page.setRequestInterception(true); //允许拦截
 			await page.evaluateOnNewDocument(() => {
 				const newProto = navigator.__proto__;
@@ -151,8 +152,7 @@ module.exports = class {
 	/**
 	 * 关闭
 	 */
-	async close(act = 1) {
-
+	async close(act = 0) {
 		try {
 			if (act & 1) await this.page.close();
 			if (act & 2) await this.browser.close();
@@ -327,10 +327,10 @@ module.exports = class {
 	 * div[name=abc]
 	 * iframe:first-child
 	 * div>ul>li:nth-child(4)>a
-	 * 
+	 *
 	 * querySelector，若匹配不到返回null
 	 * page.$()，若匹配不上则会抛出错误，这里最后也会在cath中返回null
-	 * 
+	 *
 	 * @param {Object} tag
 	 */
 	async element(tag, selector = false) {
@@ -387,6 +387,7 @@ module.exports = class {
 	async waiting(time) {
 		return await this.sleep(time);
 	}
+
 	async sleep(time) {
 		if (time < 100) time = time * 1000;
 		try {
@@ -576,7 +577,6 @@ module.exports = class {
 	}
 
 
-
 	async parseResponse(response) {
 		// const response = await this.page.waitForResponse(res => res);
 
@@ -598,7 +598,7 @@ module.exports = class {
 		value.datetime = (new Date(headers['date'])).date('yyyy-mm-dd hh:ii:ss');
 		// value.headers = headers;
 		if (headers['server']) value.server = headers['server'];
-		if (headers['set-cookie']) value.cookies = await this.cookies.parseCookies(headers['set-cookie']);
+		if (headers['set-cookie']) value.cookies = await this.cookies.parse(headers['set-cookie']);
 		value.remote = await response.remoteAddress(); //目标服务器
 		if (value.status === 301 || value.status === 302) return value;
 		if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping', 'fetch'].has(value.type)) return value;
@@ -612,50 +612,48 @@ module.exports = class {
 		if (value.post) value.post = value.post.toString();
 
 		try {
-			value.buffer = await response.buffer();
-			value.response = value.buffer.toString();
+			const buffer = await response.buffer();
+			value.response = buffer.toString();
 			value.json = JSON.parse(value.response);
 		}
 		catch (e) {
-			value.json = e.parse();
+			const err = e.parse();
+			value.json = err.message;
 		}
 
 		return value;
 	}
 
+	host(url) {
+		// const domain = url.protocol + '//' + url.host;
+		const urls = parseUrl(url);
+		return '.' + urls.host.split('.').slice(-2).join('.');
+	}
+
 
-	doListening(options) {
+	async doListening(options) {
 
 		this.page.on('request', async (request) => {
+			const host = this.host(request.url());
+			const headers = request.headers();
+			if (headers.cookie) this.cookies.request(headers.cookie, host);
+
 			if (this.requestCall) {
 				const run = await this.requestCall(request);
-				if (run === false) {
-					request.abort();
-					return;
-				}
+				if (run === false) return request.abort();
 			}
 
-			const { abort_img = false, no_cache = true, headers = {} } = options;
-			if (abort_img && /\.(?:png|jpg|jpeg|svg|gif|bmp)$/i.test(request.url())) {
-				request.abort();
-				return;
-			}
+			const rType = request.resourceType();
+			const { abort = [], headers: optHead = {} } = options;
 
-			const headersAll = request.headers();
-			if (headers !== {}) Object.assign(headersAll, headers);
+			if (abort.length > 0 && abort.some(t => t === rType)) return request.abort();
 
-			headersAll['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
-			headersAll['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
-			// headersAll['Access-Control-Allow-Headers'] = 'Content-Type';
+			if (optHead !== {}) Object.assign(headers, optHead);
 
-			if (no_cache) {
-				const rType = request.resourceType();
-				if (rType === 'script' || rType === 'stylesheet') {
-					headersAll['Cache-Control'] = 'no-store'; // 禁用缓存
-				}
-			}
-
-			request.continue({ headers: headersAll });
+			headers['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
+			headers['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
+			// headers['Access-Control-Allow-Headers'] = 'Content-Type';
+			request.continue({ headers });
 		});
 
 		this.page.on('response', async res => {
@@ -664,7 +662,6 @@ module.exports = class {
 			await this.responseCall(json);
 		});
 
-
 	}
 
 }

package/src/cookies.js

@@ -1,10 +1,11 @@
 const fs = require("fs");
+const { parse: parseUrl } = require("url");
 
 
 module.exports = class {
 	page = void 0;
 	file = void 0;
-	cookies = void 0;
+	cookies = [];
 
 	constructor(page, cookies) {
 		this.page = page;
@@ -12,14 +13,16 @@ module.exports = class {
 
 		this.file = cookies;
 		if (!fs.existsSync(cookies)) return;
-		console.log('this.cookie_file', cookies);
+		// console.log('cookies file:', cookies);
 		let cookiesVal = fs.readFileSync(cookies, 'utf8');
+		if (cookiesVal === '{}') cookiesVal = '[]';
 		this.cookies = JSON.parse(cookiesVal);
+		// this.set(this.cookies);
 	}
 
-	jsonArray(cookiesVal, host) {
-		if (!cookiesVal) return [];
-		return JSON.parse(cookiesVal).map(cook => {
+	json(str, host) {
+		if (!str) return [];
+		return JSON.parse(str).map(cook => {
 			let { name, value, domain } = cook;
 			if (!domain) domain = host;
 			return { name, value, domain };
@@ -27,19 +30,19 @@ module.exports = class {
 	}
 
 	/**
-	
+
 	 // const url = parseUrl(this.page.url());
 	 // const host = '.' + url.host.split('.').slice(-2).join('.');
 	 // let cookiesVal = read(file, 'utf8');
 	 // cookiesVal = this.jsonArray(cookiesVal, host);
-	
+
 	 */
 	/**
 	 * 解析网页set-cookies的值
 	 *
 	 * @param {Object} strCookies
 	 */
-	async parseCookies(strCookies) {
+	async parse(strCookies) {
 		return await strCookies.split("\n").map((ls) => {
 			let value = {};
 			ls.split(';').map((ln, j) => {
@@ -54,6 +57,8 @@ module.exports = class {
 				}
 				else if (Key === 'Max-Age') {
 					value.expire = parseInt(arr[1]) + (Date.now() / 1000);
+					if (value.expire > 0) value.expire_date = (value.expire * 1000).date('Y-m-d H:i:s.SSS')
+					//有可能存在expires
 				}
 				else if (Key === 'Secure') {
 					value.source = true;
@@ -62,37 +67,29 @@ module.exports = class {
 				else {
 					value[Key[0].toLowerCase() + Key.substring(1)] = arr[1] || true;
 				}
+
+				if (value.expires && value.expires > 0) {
+					value.expires_date = (value.expires * 1000).date('YYYY-mm-dd HH:ii:ss.SSS')
+				}
+
 			})
 			return value;
 		})
 	}
 
-	/**
-	 * 合并两个Cookies，并以后面的值为准
-	 *
-	 * @param {Object} oldCookies
-	 * @param {Object} newCookies
-	 */
-	async mergeNewCookies(oldCookies, newCookies) {
-		if (newCookies.length === 0) return oldCookies;
-		let aMap = new Map(oldCookies.map(obj => [obj.name, obj]));
-		// let aMap = JSON.parse(JSON.stringify(this.cookies)); //深拷贝
-		newCookies.map(obj => {
-			aMap.set(obj.name, obj)
-		});
-		return await Array.from(aMap.values());
-	}
-
 
 	/**
 	 * 获取当前页面的Cookies
 	 */
-	async getCookies() {
+	async get(key) {
 		try {
-			return await this.page.cookies();
+			await this.merge();
+			// const cookies = await this.page.cookies();
+			if (key === undefined) return this.cookies;
+			return (this.cookies.filter(c => c.name === key) || [{}])[0];
 		}
 		catch (e) {
-			console.log('[chrome.getCookies.Error]', e.message);
+			console.log('[chrome.cookies.get.Error]', e.message);
 			return [];
 		}
 	}
@@ -101,38 +98,91 @@ module.exports = class {
 	 * 设置Cookies
 	 * @param {Object} cookies
 	 */
-	async setCookies(cookies) {
+	async set(cookies) {
 		try {
+			if (cookies === undefined) cookies = this.cookies;
 			await this.page.setCookie(...cookies.map(ck => {
 				if (typeof ck.expires === 'string') ck.expires = new Date(ck.expires).getTime();
 				return ck;
 			}).filter(ck => (!!ck.domain || !!ck.url)));
 		}
 		catch (e) {
-			console.log('[chrome.setCookies.Error]', e.message);
+			console.log('[chrome.cookies.set.Error]', e.message);
 			console.log(JSON.stringify(cookies));
 		}
 	}
 
 	/**
 	 * 保存当前页面中的Cookies
-	 * @param {Object} file
 	 */
-	async saveCookies(file, append = true) {
+	async save() {
+		try {
+			if (!this.file) return [];
+			await this.merge();
+			await fs.writeFileSync(this.file, JSON.stringify(this.cookies, null, 2));
+			return this.cookies;
+		}
+		catch (e) {
+			console.log('[chrome.cookies.save.Error]', e.message);
+			return [];
+		}
+	}
+
+	async merge() {
 		try {
 			let cookies = await this.page.cookies();
+			// console.log('merge>>>>', cookies, '<<<<<merge');
+			if (cookies.length === 0) return;
+
+			const newMap = new Map(this.cookies.map(obj => [obj.name, obj]));
+			cookies.map(obj => {
+				if (obj.expires && obj.expires > 0) obj.expires_date = (obj.expires * 1000).date('yyyy-mm-dd HH:ii:ss.SSS')
+				newMap.set(obj.name, obj)
+			});
+			this.cookies = await Array.from(newMap.values());
 
-			if (append && fs.existsSync(file)) {
-				let dbCookies = JSON.parse(fs.readFileSync(file, 'utf8'));
-				cookies = this.mergeNewCookies(dbCookies, cookies);
-			}
-			await fs.writeFileSync(file, JSON.stringify(cookies, null, 2));
 		}
 		catch (e) {
-			console.log('[chrome.saveCookies.Error]', e.message);
+			console.log('[chrome.cookies.merge.Error]', e.message);
 		}
 	}
 
+	/**
+	 * 合并request时自动带上的cookies
+	 *
+	 * @param cookie
+	 * @param domain
+	 * @returns {*[]|*}
+	 */
+	async request(cookie, domain) {
+		if (!cookie) return [];
+		const cookies = cookie.split(';').map(ck => {
+			const [name, value] = ck.trim().split('=');
+			return { name, value };
+		});
+		const newMap = new Map(this.cookies.map(obj => [obj.name, obj]));
+		cookies.map(obj => {
+			const { name, value } = obj;
+			if (!newMap.has(obj.name)) {
+				newMap.set(obj.name, { name, value, domain })
+				return;
+			}
+			const cookies = newMap.get(obj.name);
+			if (!cookies.domain) {
+				newMap.set(obj.name, { name, value, domain })
+				return;
+			}
+			if (cookies.value === value) return;
+
+			const host = '.' + cookies.domain.split('.').slice(-2).join('.');
+			if (host === domain) {
+				cookies.value = value;
+				newMap.set(obj.name, cookies)
+			}
+		});
+
+		this.cookies = await Array.from(newMap.values());
+	}
 
 
 }