nodejs_chromium 1.0.7 → 1.1.0

package/index.js

@@ -6,56 +6,66 @@ global.__UA__ = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (K
 async function newChrome(params) {
 	let {
 		id = 'myChrome',
-			visible = false,
-			width = 1024,
-			height = 768,
-			scale = 1,
-			mobile = false,
-			dumpio = false,
-			debug = false,
-			incognito = true,
-			path = void 0,
-			slowMo = 1,
-			ua = void 0
+			visible = false, //是否打开浏览器
+			width = 1024, //宽
+			height = 768, //高
+			scale = 1, //缩放比例
+			mobile = false, //手机版
+			dumpio = false, //转发标准输入/输出流
+			devtools = false, //打开devtools
+			cache = false, //禁用缓存
+			incognito = true, //使用无痕模式启动
+			path = void 0, //Chrome路径，linux下必填
+			slowMo = 1, //每一步停留时间
+			ua = void 0, //若不指定ua则用__UA__
+			proxy = null, //代理，如：https://proxy.com:789/
+			cookies = null, //若cookies=false，则不处理cookies，不指定则由chrome处理，若=文件路径
+			abort = [], //要禁止的类型，常见["font","image","ping","stylesheet","document","fetch","script","xhr"]
+			headers = {},
 	} = params;
 
 	let option = {
-		userDataDir: `runtime/template/${id}`,
+		userDataDir: `runtime/.cache/${id}`,
 		// timeout: 500,//最大允许超时间，默认为3000，最小215，一般不要设置
 		slowMo, //每一步停留时间，不能太大，否则会太慢，特别是在类似写入很多Cookies时，每写入一个都要等一下。
 		headless: visible ? false : 'new', //'new',设置是否在无头模式下运行浏览器,false=会启动浏览器，true=无界面
-		devtools: !!debug, //打开调试
+		devtools: !!devtools, //打开调试
 		ignoreHTTPSErrors: true, //忽略 HTTPS 错误。屏蔽跳转不同域名的报错
-		ignoreDefaultArgs: [
-			"--enable-automation", //忽略默认的 --enable-automation 参数，这有助于防止某些网站检测到自动化行为。
-		],
-		dumpio: !!dumpio, //是否将浏览器的标准输入/输出流（stdio）的内容输出到 Node.js 的 stdout 和 stderr。
+		dumpio, //是否将浏览器的标准输入/输出流（stdio）的内容输出到 Node.js 的 stdout 和 stderr。
 		defaultViewport: {
 			width,
 			height,
 			deviceScaleFactor: scale, //缩放比例
 			isMobile: !!mobile,
 		},
+		defaultArgs: [
+			'--disable-extensions', //禁止启动扩展
+			'--no-sandbox', //禁用沙箱模式
+			'--disable-setuid-sandbox', //禁用设置用户身份沙盒
+			'--disable-web-security', //禁用同源策略
+		],
+		ignoreDefaultArgs: [
+			"--enable-automation", //忽略默认的 --enable-automation 参数，这有助于防止某些网站检测到自动化行为。
+		],
 		args: [
 			'--enable-chrome-browser-cloud-management', //Cloud Browser Client Management (CBCM)
 			'--disable-web-security', //禁用浏览器的同源策略（Same-Origin Policy）和跨站请求伪造（CSRF）保护
 			`--window-size=${width},${height}`,
 			'--no-sandbox', //禁用沙箱模式
+			'--disable-autofill-backend', //禁止自动填充
 			'--disable-setuid-sandbox', //禁用 setuid 沙箱。这是另一种沙箱模式，通常用于 Linux 系统上。
-			'--disable-infobars', //禁用 Chrome 在自动化控制时显示的信息栏。这个信息栏通常会告诉用户浏览器正在被自动化工具控制。
+			// '--disable-infobars', //禁用自动化控制时显示的信息栏，告诉用户浏览器正在被自动化工具控制。
 			'--disable-gpu', //禁用 GPU 加速。这通常用于在服务器环境或某些不支持 GPU 加速的平台上运行 Chrome。
-			'--disable-blink-features=AutomationControlled', //禁用 blink 引擎的自动化控制特性，防止网站通过检测 blink 引擎的特性来判断浏览器是否处于自动化控制状态。
-			// 在自动化测试或爬虫等场景中，这个参数可以帮助隐藏浏览器的自动化痕迹，使得浏览器行为更接近于真实用户操作
+			'--disable-blink-features=AutomationControlled', //防止检测 blink 引擎特性判断是否处于自动化控制状态。
 			'--lang=zh-CN', //设置中文环境
 			'--disable-extensions', //禁止启动扩展
 			'--disable-dev-shm-usage', //Linux系统中使用普通的文件系统缓存避免因为/dev/shm大小不足而导致的问题
 		]
 	}
 
-	if (incognito) { //使用无痕模式启动
-		option.args.push('--incognito', '--disable-infobars')
-	}
-
+	if (proxy) option.defaultArgs.push(`--proxy-server=${proxy}`);
+	if (incognito) option.args.push('--incognito', '--disable-infobars'); //使用无痕模式启动
+	if (!cache) option.args.push('--disable-cache'); //禁缓存
 	if (path) option.executablePath = path; //指定chrome安装路径
 
 	const browser = await puppeteer.launch(option);
@@ -65,11 +75,13 @@ async function newChrome(params) {
 	if (ua) await page.setUserAgent(ua);
 	await page.evaluateOnNewDocument(() => {
 		const newProto = navigator.__proto__;
-		delete newProto.webdriver; //删除 navigator.webdriver字段
+		delete newProto.webdriver; //删除 navigator.webdriver字段，防止检测到自动化行为
 		navigator.__proto__ = newProto;
-	}); //在每次新文档加载时，删除 navigator.webdriver 字段，这有助于防止某些网站检测到自动化行为。
+	});
 
-	return new chrome(browser, page, params);
+	if (cookies !== false) cookies = `runtime/.cache/${id}/cookies.json`;
+	const pageOption = { cookies, visible, abort, headers };
+	return new chrome(browser, page, pageOption);
 }
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodejs_chromium",
-  "version": "1.0.7",
+  "version": "1.1.0",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/chrome.js

@@ -1,4 +1,5 @@
 const fs = require("fs");
+const Cookies = require("./cookies");
 const { parse: parseUrl } = require("url");
 
 
@@ -12,21 +13,23 @@ const { parse: parseUrl } = require("url");
  * @type {exports}
  */
 module.exports = class {
-	browser = null;
-	page = null;
-	responseCall = null;
-	requestCall = null;
+	browser = void 0;
+	page = void 0;
+	responseCall = void 0;
+	requestCall = void 0;
+	cookies = void 0;
+	options = void 0;
 	isFrame = false; //是不是在iFrame中
 	visible = false; //是否可见，也就是有没有启动窗口
-	params = {};
 
-	constructor(browser, page, params, isFrame = false) {
+	constructor(browser, page, options, isFrame = false) {
 		this.browser = browser;
 		this.page = page;
-		this.params = params;
-		this.visible = !!params.visible;
+		this.options = options;
+		this.cookies = new Cookies(page, options.cookies);
+		this.visible = !!options.visible;
 		this.isFrame = !!isFrame;
-		this.doListening(params);
+		this.doListening(options);
 	}
 
 	request(call) {
@@ -45,14 +48,14 @@ module.exports = class {
 	async clone() {
 		try {
 			const page = await this.browser.newPage();
+			await page.setBypassCSP(true); //绕过页面的内容安全策略
 			await page.setRequestInterception(true); //允许拦截
-			// await page.setUserAgent(this.params.ua);
 			await page.evaluateOnNewDocument(() => {
 				const newProto = navigator.__proto__;
 				delete newProto.webdriver; //删除 navigator.webdriver字段
 				navigator.__proto__ = newProto; //在每次新文档加载时，删除 navigator.webdriver 字段，这有助于防止某些网站检测到自动化行为。
 			});
-			return new module.exports(this.browser, page, this.params, false); //new 自身
+			return new module.exports(this.browser, page, this.options, false); //new 自身
 		}
 		catch (e) {
 			console.log('[chrome.iframe.Error]', e.message);
@@ -139,8 +142,7 @@ module.exports = class {
 	async iframe(tag) {
 		try {
 			const frame = await (await this.page.$(tag)).contentFrame();
-			// return new iframe(this.browser, frame, this.params);
-			return new module.exports(this.browser, frame, this.params, true); //new 自身
+			return new module.exports(this.browser, frame, this.options, true); //new 自身
 		}
 		catch (e) {
 			console.log('[chrome.iframe.Error]', e.message);
@@ -150,8 +152,7 @@ module.exports = class {
 	/**
 	 * 关闭
 	 */
-	async close(act = 3) {
-
+	async close(act = 0) {
 		try {
 			if (act & 1) await this.page.close();
 			if (act & 2) await this.browser.close();
@@ -327,10 +328,14 @@ module.exports = class {
 	 * iframe:first-child
 	 * div>ul>li:nth-child(4)>a
 	 *
+	 * querySelector，若匹配不到返回null
+	 * page.$()，若匹配不上则会抛出错误，这里最后也会在cath中返回null
+	 *
 	 * @param {Object} tag
 	 */
-	async element(tag) {
+	async element(tag, selector = false) {
 		try {
+			if (selector) return await this.page.querySelector(tag);
 			return await this.page.$(tag);
 		}
 		catch (e) {
@@ -382,6 +387,7 @@ module.exports = class {
 	async waiting(time) {
 		return await this.sleep(time);
 	}
+
 	async sleep(time) {
 		if (time < 100) time = time * 1000;
 		try {
@@ -495,90 +501,6 @@ module.exports = class {
 	}
 
 
-	jsonArray(cookiesVal, host) {
-		if (!cookiesVal) return [];
-		return JSON.parse(cookiesVal).map(cook => {
-			let { name, value, domain } = cook;
-			if (!domain) domain = host;
-			return { name, value, domain };
-		});
-	}
-
-	/**
-
-	 // const url = parseUrl(this.page.url());
-	 // const host = '.' + url.host.split('.').slice(-2).join('.');
-	 // let cookiesVal = read(file, 'utf8');
-	 // cookiesVal = this.jsonArray(cookiesVal, host);
-
-	 */
-
-	/**
-	 * 合并两个Cookies，并以后面的值为准
-	 *
-	 * @param {Object} oldCookies
-	 * @param {Object} newCookies
-	 */
-	async mergeNewCookies(oldCookies, newCookies) {
-		if (newCookies.length === 0) return oldCookies;
-		// let aMap = new Map(oldCookies.map(obj => [obj.name, obj]));//浅拷贝
-		let aMap = JSON.parse(JSON.stringify(this.cookies)); //深拷贝
-		newCookies.map(obj => {
-			aMap.set(obj.name, obj)
-		});
-		return await Array.from(aMap.values());
-	}
-
-
-	/**
-	 * 获取当前页面的Cookies
-	 */
-	async getCookies() {
-		try {
-			return await this.page.cookies();
-		}
-		catch (e) {
-			console.log('[chrome.getCookies.Error]', e.message);
-			return [];
-		}
-	}
-
-	/**
-	 * 设置Cookies
-	 * @param {Object} cookies
-	 */
-	async setCookies(cookies) {
-		try {
-			await this.page.setCookie(...cookies);
-			return this;
-		}
-		catch (e) {
-			console.log('[chrome.setCookies.Error]', e.message);
-			console.log(cookies);
-		}
-	}
-
-	/**
-	 * 保存当前页面中的Cookies
-	 * @param {Object} file
-	 */
-	async saveCookies(file, append = true) {
-		try {
-			let cookies = await this.page.cookies();
-
-			if (append && fs.existsSync(file)) {
-				let dbCookies = JSON.parse(fs.readFileSync(file, 'utf8'));
-				cookies = this.mergeNewCookies(dbCookies, cookies);
-			}
-
-			await fs.writeFileSync(file, JSON.stringify(cookies, null, 2));
-			return this;
-		}
-		catch (e) {
-			console.log('[chrome.saveCookies.Error]', e.message);
-		}
-	}
-
 	/**
 	 * 加水印，这里实际上是在网页加个DIV并显示时间
 	 * @param {Object} conf
@@ -633,20 +555,20 @@ module.exports = class {
 	}
 
 	/**
-	 * 补全所有本地js/css
+	 * 补全所有本地js/css，一般用于保存html之前
 	 */
 	async improveUrls() {
 		const url = parseUrl(this.page.url());
 		const domain = url.protocol + '//' + url.host;
 		await this.page.evaluate((domain) => {
 			try {
-				const tags = document.querySelectorAll('script[src], link[href], iframe[src]');
-				tags.forEach((ele) => {
-					const src = ele.getAttribute('src');
-					const href = ele.getAttribute('href');
-					if (src && src.startsWith('/')) ele.src = domain + src;
-					if (href && href.startsWith('/')) ele.href = domain + href;
-				});
+				document.querySelectorAll('script[src], link[href], iframe[src]')
+					.forEach((ele) => {
+						const src = ele.getAttribute('src');
+						const href = ele.getAttribute('href');
+						if (src && src.startsWith('/')) ele.src = domain + src;
+						if (href && href.startsWith('/')) ele.href = domain + href;
+					});
 			}
 			catch (e) {
 				console.log('[chrome.improveUrls.Error]', e.message);
@@ -655,40 +577,6 @@ module.exports = class {
 	}
 
 
-	/**
-	 * 解析网页set-cookies的值
-	 *
-	 * @param {Object} strCookies
-	 */
-	async parseCookies(strCookies) {
-		return await strCookies.split("\n").map((ls) => {
-			let value = {};
-			ls.split(';').map((ln, j) => {
-				// console.log(ln);
-				const arr = ln.split('=');
-				const Key = (arr[0]).trim();
-				if (!Key) return;
-
-				if (j === 0) {
-					value.name = Key;
-					value.value = arr[1];
-				}
-				else if (Key === 'Max-Age') {
-					value.expire = parseInt(arr[1]) + (Date.now() / 1000);
-				}
-				else if (Key === 'Secure') {
-					value.source = true;
-					value.sourceScheme = 'Secure';
-				}
-				else {
-					value[Key[0].toLowerCase() + Key.substring(1)] = arr[1] || true;
-				}
-			})
-			return value;
-		})
-	}
-
-
 	async parseResponse(response) {
 		// const response = await this.page.waitForResponse(res => res);
 
@@ -710,7 +598,7 @@ module.exports = class {
 		value.datetime = (new Date(headers['date'])).date('yyyy-mm-dd hh:ii:ss');
 		// value.headers = headers;
 		if (headers['server']) value.server = headers['server'];
-		if (headers['set-cookie']) value.cookies = await this.parseCookies(headers['set-cookie']);
+		if (headers['set-cookie']) value.cookies = await this.cookies.parse(headers['set-cookie']);
 		value.remote = await response.remoteAddress(); //目标服务器
 		if (value.status === 301 || value.status === 302) return value;
 		if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping', 'fetch'].has(value.type)) return value;
@@ -724,49 +612,47 @@ module.exports = class {
 		if (value.post) value.post = value.post.toString();
 
 		try {
-			value.buffer = await response.buffer();
-			value.response = value.buffer.toString();
+			const buffer = await response.buffer();
+			value.response = buffer.toString();
 			value.json = JSON.parse(value.response);
 		}
 		catch (e) {
-			value.json = e.parse();
+			const err = e.parse();
+			value.json = err.message;
 		}
 
 		return value;
 	}
 
+	host(url) {
+		// const domain = url.protocol + '//' + url.host;
+		const urls = parseUrl(url);
+		return '.' + urls.host.split('.').slice(-2).join('.');
+	}
 
-	doListening(params) {
+
+	async doListening(options) {
 
 		this.page.on('request', async (request) => {
+			const host = this.host(request.url());
+			const headers = request.headers();
+			if (headers.cookie) this.cookies.request(headers.cookie, host);
+
 			if (this.requestCall) {
 				const run = await this.requestCall(request);
-				if (run === false) {
-					request.abort();
-					return;
-				}
+				if (run === false) return request.abort();
 			}
 
-			const { abort_img = false, no_cache = true, append_headers = {} } = params;
-			if (abort_img && /\.(?:png|jpg|jpeg|svg|gif)$/i.test(request.url())) {
-				request.abort();
-				return;
-			}
+			const rType = request.resourceType();
+			const { abort = [], headers: optHead = {} } = options;
 
-			const headers = request.headers();
-			if (append_headers !== {}) Object.assign(headers, append_headers);
+			if (abort.length > 0 && abort.some(t => t === rType)) return request.abort();
+
+			if (optHead !== {}) Object.assign(headers, optHead);
 
 			headers['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
 			headers['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
 			// headers['Access-Control-Allow-Headers'] = 'Content-Type';
-
-			if (no_cache) {
-				const rType = request.resourceType();
-				if (rType === 'script' || rType === 'stylesheet') {
-					headers['Cache-Control'] = 'no-store'; // 禁用缓存
-				}
-			}
-
 			request.continue({ headers });
 		});
 
@@ -775,7 +661,7 @@ module.exports = class {
 			let json = await this.parseResponse(res);
 			await this.responseCall(json);
 		});
-	}
 
+	}
 
 }

package/src/cookies.js

@@ -0,0 +1,179 @@
+const fs = require("fs");
+const {parse: parseUrl} = require("url");
+
+
+module.exports = class {
+    page = void 0;
+    file = void 0;
+    cookies = [];
+
+    constructor(page, cookies) {
+        this.page = page;
+        if (cookies === false) return;
+
+        this.file = cookies;
+        if (!fs.existsSync(cookies)) return;
+        // console.log('cookies file:', cookies);
+        let cookiesVal = fs.readFileSync(cookies, 'utf8');
+        if (cookiesVal === '{}') cookiesVal = '[]';
+        this.cookies = JSON.parse(cookiesVal);
+        // this.set(this.cookies);
+    }
+
+    json(str, host) {
+        if (!str) return [];
+        return JSON.parse(str).map(cook => {
+            let {name, value, domain} = cook;
+            if (!domain) domain = host;
+            return {name, value, domain};
+        });
+    }
+
+    /**
+
+     // const url = parseUrl(this.page.url());
+     // const host = '.' + url.host.split('.').slice(-2).join('.');
+     // let cookiesVal = read(file, 'utf8');
+     // cookiesVal = this.jsonArray(cookiesVal, host);
+
+     */
+    /**
+     * 解析网页set-cookies的值
+     *
+     * @param {Object} strCookies
+     */
+    async parse(strCookies) {
+        return await strCookies.split("\n").map((ls) => {
+            let value = {};
+            ls.split(';').map((ln, j) => {
+                // console.log(ln);
+                const arr = ln.split('=');
+                const Key = (arr[0]).trim();
+                if (!Key) return;
+
+                if (j === 0) {
+                    value.name = Key;
+                    value.value = arr[1];
+                } else if (Key === 'Max-Age') {
+                    value.expire = parseInt(arr[1]) + (Date.now() / 1000);
+                    if (value.expire > 0) value.expire_date = (value.expire * 1000).date('Y-m-d H:i:s.SSS')
+                    //有可能存在expires
+                } else if (Key === 'Secure') {
+                    value.source = true;
+                    value.sourceScheme = 'Secure';
+                } else {
+                    value[Key[0].toLowerCase() + Key.substring(1)] = arr[1] || true;
+                }
+
+                if (value.expires && value.expires > 0) {
+                    value.expires_date = (value.expires * 1000).date('YYYY-mm-dd HH:ii:ss.SSS')
+                }
+
+            })
+            return value;
+        })
+    }
+
+
+    /**
+     * 获取当前页面的Cookies
+     */
+    async get(key) {
+        try {
+            await this.merge();
+            // const cookies = await this.page.cookies();
+            if (key === undefined) return this.cookies;
+            return (this.cookies.filter(c => c.name === key) || [{}])[0];
+        } catch (e) {
+            console.log('[chrome.cookies.get.Error]', e.message);
+            return [];
+        }
+    }
+
+    /**
+     * 设置Cookies
+     * @param {Object} cookies
+     */
+    async set(cookies) {
+        try {
+            if (cookies === undefined) cookies = this.cookies;
+            await this.page.setCookie(...cookies.map(ck => {
+                if (typeof ck.expires === 'string') ck.expires = new Date(ck.expires).getTime();
+                return ck;
+            }).filter(ck => (!!ck.domain || !!ck.url)));
+        } catch (e) {
+            console.log('[chrome.cookies.set.Error]', e.message);
+            console.log(JSON.stringify(cookies));
+        }
+    }
+
+    /**
+     * 保存当前页面中的Cookies
+     */
+    async save() {
+        try {
+            if (!this.file) return;
+            await this.merge();
+            await fs.writeFileSync(this.file, JSON.stringify(this.cookies, null, 2));
+        } catch (e) {
+            console.log('[chrome.cookies.save.Error]', e.message);
+        }
+    }
+
+    async merge() {
+        try {
+            let cookies = await this.page.cookies();
+            // console.log('merge>>>>', cookies, '<<<<<merge');
+            if (cookies.length === 0) return;
+
+            const newMap = new Map(this.cookies.map(obj => [obj.name, obj]));
+            cookies.map(obj => {
+                if (obj.expires && obj.expires > 0) obj.expires_date = (obj.expires * 1000).date('yyyy-mm-dd HH:ii:ss.SSS')
+                newMap.set(obj.name, obj)
+            });
+            this.cookies = await Array.from(newMap.values());
+
+        } catch (e) {
+            console.log('[chrome.cookies.merge.Error]', e.message);
+        }
+    }
+
+    /**
+     * 合并request时自动带上的cookies
+     *
+     * @param cookie
+     * @param domain
+     * @returns {*[]|*}
+     */
+    async request(cookie, domain) {
+        if (!cookie) return [];
+        const cookies = cookie.split(';').map(ck => {
+            const [name, value] = ck.trim().split('=');
+            return {name, value};
+        });
+        const newMap = new Map(this.cookies.map(obj => [obj.name, obj]));
+        cookies.map(obj => {
+            const {name, value} = obj;
+            if (!newMap.has(obj.name)) {
+                newMap.set(obj.name, {name, value, domain})
+                return;
+            }
+            const cookies = newMap.get(obj.name);
+            if (!cookies.domain) {
+                newMap.set(obj.name, {name, value, domain})
+                return;
+            }
+            if (cookies.value === value) return;
+
+            const host = '.' + cookies.domain.split('.').slice(-2).join('.');
+            if (host === domain) {
+                cookies.value = value;
+                newMap.set(obj.name, cookies)
+            }
+        });
+
+        this.cookies = await Array.from(newMap.values());
+    }
+
+
+}