nodejs_chromium 1.0.7 → 1.0.8

package/index.js

@@ -16,7 +16,12 @@ async function newChrome(params) {
 			incognito = true,
 			path = void 0,
 			slowMo = 1,
-			ua = void 0
+			ua = void 0,
+			proxy = null,
+			cookies = null, //若cookies=false，则不处理cookies，不指定则由chrome处理，若=文件路径
+			abort_img = false,
+			no_cache = true,
+			headers = {},
 	} = params;
 
 	let option = {
@@ -26,9 +31,6 @@ async function newChrome(params) {
 		headless: visible ? false : 'new', //'new',设置是否在无头模式下运行浏览器,false=会启动浏览器，true=无界面
 		devtools: !!debug, //打开调试
 		ignoreHTTPSErrors: true, //忽略 HTTPS 错误。屏蔽跳转不同域名的报错
-		ignoreDefaultArgs: [
-			"--enable-automation", //忽略默认的 --enable-automation 参数，这有助于防止某些网站检测到自动化行为。
-		],
 		dumpio: !!dumpio, //是否将浏览器的标准输入/输出流（stdio）的内容输出到 Node.js 的 stdout 和 stderr。
 		defaultViewport: {
 			width,
@@ -36,26 +38,32 @@ async function newChrome(params) {
 			deviceScaleFactor: scale, //缩放比例
 			isMobile: !!mobile,
 		},
+		defaultArgs: [
+			'--disable-extensions', //禁止启动扩展
+			'--no-sandbox', //禁用沙箱模式
+			'--disable-setuid-sandbox', //禁用设置用户身份沙盒
+			'--disable-web-security', //禁用同源策略
+		],
+		ignoreDefaultArgs: [
+			"--enable-automation", //忽略默认的 --enable-automation 参数，这有助于防止某些网站检测到自动化行为。
+		],
 		args: [
 			'--enable-chrome-browser-cloud-management', //Cloud Browser Client Management (CBCM)
 			'--disable-web-security', //禁用浏览器的同源策略（Same-Origin Policy）和跨站请求伪造（CSRF）保护
 			`--window-size=${width},${height}`,
 			'--no-sandbox', //禁用沙箱模式
 			'--disable-setuid-sandbox', //禁用 setuid 沙箱。这是另一种沙箱模式，通常用于 Linux 系统上。
-			'--disable-infobars', //禁用 Chrome 在自动化控制时显示的信息栏。这个信息栏通常会告诉用户浏览器正在被自动化工具控制。
+			// '--disable-infobars', //禁用 Chrome 在自动化控制时显示的信息栏。这个信息栏通常会告诉用户浏览器正在被自动化工具控制。
 			'--disable-gpu', //禁用 GPU 加速。这通常用于在服务器环境或某些不支持 GPU 加速的平台上运行 Chrome。
-			'--disable-blink-features=AutomationControlled', //禁用 blink 引擎的自动化控制特性，防止网站通过检测 blink 引擎的特性来判断浏览器是否处于自动化控制状态。
-			// 在自动化测试或爬虫等场景中，这个参数可以帮助隐藏浏览器的自动化痕迹，使得浏览器行为更接近于真实用户操作
+			'--disable-blink-features=AutomationControlled', //防止检测 blink 引擎特性判断是否处于自动化控制状态。
 			'--lang=zh-CN', //设置中文环境
 			'--disable-extensions', //禁止启动扩展
 			'--disable-dev-shm-usage', //Linux系统中使用普通的文件系统缓存避免因为/dev/shm大小不足而导致的问题
 		]
 	}
 
-	if (incognito) { //使用无痕模式启动
-		option.args.push('--incognito', '--disable-infobars')
-	}
-
+	if (proxy) option.defaultArgs.push(`--proxy-server=${proxy}`);
+	if (incognito) option.args.push('--incognito', '--disable-infobars'); //使用无痕模式启动
 	if (path) option.executablePath = path; //指定chrome安装路径
 
 	const browser = await puppeteer.launch(option);
@@ -65,11 +73,13 @@ async function newChrome(params) {
 	if (ua) await page.setUserAgent(ua);
 	await page.evaluateOnNewDocument(() => {
 		const newProto = navigator.__proto__;
-		delete newProto.webdriver; //删除 navigator.webdriver字段
+		delete newProto.webdriver; //删除 navigator.webdriver字段，防止检测到自动化行为
 		navigator.__proto__ = newProto;
-	}); //在每次新文档加载时，删除 navigator.webdriver 字段，这有助于防止某些网站检测到自动化行为。
+	});
 
-	return new chrome(browser, page, params);
+	if (cookies !== false) cookies = `runtime/cookies/${id}`;
+	const pageOption = { cookies, visible, abort_img, no_cache, headers };
+	return new chrome(browser, page, pageOption);
 }
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodejs_chromium",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "",
   "main": "index.js",
   "scripts": {

package/src/chrome.js

@@ -1,4 +1,5 @@
 const fs = require("fs");
+const Cookies = require("./cookies");
 const { parse: parseUrl } = require("url");
 
 
@@ -12,21 +13,23 @@ const { parse: parseUrl } = require("url");
  * @type {exports}
  */
 module.exports = class {
-	browser = null;
-	page = null;
-	responseCall = null;
-	requestCall = null;
+	browser = void 0;
+	page = void 0;
+	responseCall = void 0;
+	requestCall = void 0;
+	cookies = void 0;
+	options = void 0;
 	isFrame = false; //是不是在iFrame中
 	visible = false; //是否可见，也就是有没有启动窗口
-	params = {};
 
-	constructor(browser, page, params, isFrame = false) {
+	constructor(browser, page, options, isFrame = false) {
 		this.browser = browser;
 		this.page = page;
-		this.params = params;
-		this.visible = !!params.visible;
+		this.options = options;
+		this.cookies = new Cookies(page, options.cookies);
+		this.visible = !!options.visible;
 		this.isFrame = !!isFrame;
-		this.doListening(params);
+		this.doListening(options);
 	}
 
 	request(call) {
@@ -46,13 +49,12 @@ module.exports = class {
 		try {
 			const page = await this.browser.newPage();
 			await page.setRequestInterception(true); //允许拦截
-			// await page.setUserAgent(this.params.ua);
 			await page.evaluateOnNewDocument(() => {
 				const newProto = navigator.__proto__;
 				delete newProto.webdriver; //删除 navigator.webdriver字段
 				navigator.__proto__ = newProto; //在每次新文档加载时，删除 navigator.webdriver 字段，这有助于防止某些网站检测到自动化行为。
 			});
-			return new module.exports(this.browser, page, this.params, false); //new 自身
+			return new module.exports(this.browser, page, this.options, false); //new 自身
 		}
 		catch (e) {
 			console.log('[chrome.iframe.Error]', e.message);
@@ -139,8 +141,7 @@ module.exports = class {
 	async iframe(tag) {
 		try {
 			const frame = await (await this.page.$(tag)).contentFrame();
-			// return new iframe(this.browser, frame, this.params);
-			return new module.exports(this.browser, frame, this.params, true); //new 自身
+			return new module.exports(this.browser, frame, this.options, true); //new 自身
 		}
 		catch (e) {
 			console.log('[chrome.iframe.Error]', e.message);
@@ -150,7 +151,7 @@ module.exports = class {
 	/**
 	 * 关闭
 	 */
-	async close(act = 3) {
+	async close(act = 1) {
 
 		try {
 			if (act & 1) await this.page.close();
@@ -326,11 +327,15 @@ module.exports = class {
 	 * div[name=abc]
 	 * iframe:first-child
 	 * div>ul>li:nth-child(4)>a
-	 *
+	 * 
+	 * querySelector，若匹配不到返回null
+	 * page.$()，若匹配不上则会抛出错误，这里最后也会在cath中返回null
+	 * 
 	 * @param {Object} tag
 	 */
-	async element(tag) {
+	async element(tag, selector = false) {
 		try {
+			if (selector) return await this.page.querySelector(tag);
 			return await this.page.$(tag);
 		}
 		catch (e) {
@@ -495,90 +500,6 @@ module.exports = class {
 	}
 
 
-	jsonArray(cookiesVal, host) {
-		if (!cookiesVal) return [];
-		return JSON.parse(cookiesVal).map(cook => {
-			let { name, value, domain } = cook;
-			if (!domain) domain = host;
-			return { name, value, domain };
-		});
-	}
-
-	/**
-
-	 // const url = parseUrl(this.page.url());
-	 // const host = '.' + url.host.split('.').slice(-2).join('.');
-	 // let cookiesVal = read(file, 'utf8');
-	 // cookiesVal = this.jsonArray(cookiesVal, host);
-
-	 */
-
-	/**
-	 * 合并两个Cookies，并以后面的值为准
-	 *
-	 * @param {Object} oldCookies
-	 * @param {Object} newCookies
-	 */
-	async mergeNewCookies(oldCookies, newCookies) {
-		if (newCookies.length === 0) return oldCookies;
-		// let aMap = new Map(oldCookies.map(obj => [obj.name, obj]));//浅拷贝
-		let aMap = JSON.parse(JSON.stringify(this.cookies)); //深拷贝
-		newCookies.map(obj => {
-			aMap.set(obj.name, obj)
-		});
-		return await Array.from(aMap.values());
-	}
-
-
-	/**
-	 * 获取当前页面的Cookies
-	 */
-	async getCookies() {
-		try {
-			return await this.page.cookies();
-		}
-		catch (e) {
-			console.log('[chrome.getCookies.Error]', e.message);
-			return [];
-		}
-	}
-
-	/**
-	 * 设置Cookies
-	 * @param {Object} cookies
-	 */
-	async setCookies(cookies) {
-		try {
-			await this.page.setCookie(...cookies);
-			return this;
-		}
-		catch (e) {
-			console.log('[chrome.setCookies.Error]', e.message);
-			console.log(cookies);
-		}
-	}
-
-	/**
-	 * 保存当前页面中的Cookies
-	 * @param {Object} file
-	 */
-	async saveCookies(file, append = true) {
-		try {
-			let cookies = await this.page.cookies();
-
-			if (append && fs.existsSync(file)) {
-				let dbCookies = JSON.parse(fs.readFileSync(file, 'utf8'));
-				cookies = this.mergeNewCookies(dbCookies, cookies);
-			}
-
-			await fs.writeFileSync(file, JSON.stringify(cookies, null, 2));
-			return this;
-		}
-		catch (e) {
-			console.log('[chrome.saveCookies.Error]', e.message);
-		}
-	}
-
 	/**
 	 * 加水印，这里实际上是在网页加个DIV并显示时间
 	 * @param {Object} conf
@@ -633,20 +554,20 @@ module.exports = class {
 	}
 
 	/**
-	 * 补全所有本地js/css
+	 * 补全所有本地js/css，一般用于保存html之前
 	 */
 	async improveUrls() {
 		const url = parseUrl(this.page.url());
 		const domain = url.protocol + '//' + url.host;
 		await this.page.evaluate((domain) => {
 			try {
-				const tags = document.querySelectorAll('script[src], link[href], iframe[src]');
-				tags.forEach((ele) => {
-					const src = ele.getAttribute('src');
-					const href = ele.getAttribute('href');
-					if (src && src.startsWith('/')) ele.src = domain + src;
-					if (href && href.startsWith('/')) ele.href = domain + href;
-				});
+				document.querySelectorAll('script[src], link[href], iframe[src]')
+					.forEach((ele) => {
+						const src = ele.getAttribute('src');
+						const href = ele.getAttribute('href');
+						if (src && src.startsWith('/')) ele.src = domain + src;
+						if (href && href.startsWith('/')) ele.href = domain + href;
+					});
 			}
 			catch (e) {
 				console.log('[chrome.improveUrls.Error]', e.message);
@@ -655,39 +576,6 @@ module.exports = class {
 	}
 
 
-	/**
-	 * 解析网页set-cookies的值
-	 *
-	 * @param {Object} strCookies
-	 */
-	async parseCookies(strCookies) {
-		return await strCookies.split("\n").map((ls) => {
-			let value = {};
-			ls.split(';').map((ln, j) => {
-				// console.log(ln);
-				const arr = ln.split('=');
-				const Key = (arr[0]).trim();
-				if (!Key) return;
-
-				if (j === 0) {
-					value.name = Key;
-					value.value = arr[1];
-				}
-				else if (Key === 'Max-Age') {
-					value.expire = parseInt(arr[1]) + (Date.now() / 1000);
-				}
-				else if (Key === 'Secure') {
-					value.source = true;
-					value.sourceScheme = 'Secure';
-				}
-				else {
-					value[Key[0].toLowerCase() + Key.substring(1)] = arr[1] || true;
-				}
-			})
-			return value;
-		})
-	}
-
 
 	async parseResponse(response) {
 		// const response = await this.page.waitForResponse(res => res);
@@ -710,7 +598,7 @@ module.exports = class {
 		value.datetime = (new Date(headers['date'])).date('yyyy-mm-dd hh:ii:ss');
 		// value.headers = headers;
 		if (headers['server']) value.server = headers['server'];
-		if (headers['set-cookie']) value.cookies = await this.parseCookies(headers['set-cookie']);
+		if (headers['set-cookie']) value.cookies = await this.cookies.parseCookies(headers['set-cookie']);
 		value.remote = await response.remoteAddress(); //目标服务器
 		if (value.status === 301 || value.status === 302) return value;
 		if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping', 'fetch'].has(value.type)) return value;
@@ -736,7 +624,7 @@ module.exports = class {
 	}
 
 
-	doListening(params) {
+	doListening(options) {
 
 		this.page.on('request', async (request) => {
 			if (this.requestCall) {
@@ -747,27 +635,27 @@ module.exports = class {
 				}
 			}
 
-			const { abort_img = false, no_cache = true, append_headers = {} } = params;
-			if (abort_img && /\.(?:png|jpg|jpeg|svg|gif)$/i.test(request.url())) {
+			const { abort_img = false, no_cache = true, headers = {} } = options;
+			if (abort_img && /\.(?:png|jpg|jpeg|svg|gif|bmp)$/i.test(request.url())) {
 				request.abort();
 				return;
 			}
 
-			const headers = request.headers();
-			if (append_headers !== {}) Object.assign(headers, append_headers);
+			const headersAll = request.headers();
+			if (headers !== {}) Object.assign(headersAll, headers);
 
-			headers['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
-			headers['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
-			// headers['Access-Control-Allow-Headers'] = 'Content-Type';
+			headersAll['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
+			headersAll['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
+			// headersAll['Access-Control-Allow-Headers'] = 'Content-Type';
 
 			if (no_cache) {
 				const rType = request.resourceType();
 				if (rType === 'script' || rType === 'stylesheet') {
-					headers['Cache-Control'] = 'no-store'; // 禁用缓存
+					headersAll['Cache-Control'] = 'no-store'; // 禁用缓存
 				}
 			}
 
-			request.continue({ headers });
+			request.continue({ headers: headersAll });
 		});
 
 		this.page.on('response', async res => {
@@ -775,7 +663,8 @@ module.exports = class {
 			let json = await this.parseResponse(res);
 			await this.responseCall(json);
 		});
-	}
 
 
+	}
+
 }

package/src/cookies.js

@@ -0,0 +1,138 @@
+const fs = require("fs");
+
+
+module.exports = class {
+	page = void 0;
+	file = void 0;
+	cookies = void 0;
+
+	constructor(page, cookies) {
+		this.page = page;
+		if (cookies === false) return;
+
+		this.file = cookies;
+		if (!fs.existsSync(cookies)) return;
+		console.log('this.cookie_file', cookies);
+		let cookiesVal = fs.readFileSync(cookies, 'utf8');
+		this.cookies = JSON.parse(cookiesVal);
+	}
+
+	jsonArray(cookiesVal, host) {
+		if (!cookiesVal) return [];
+		return JSON.parse(cookiesVal).map(cook => {
+			let { name, value, domain } = cook;
+			if (!domain) domain = host;
+			return { name, value, domain };
+		});
+	}
+
+	/**
+	
+	 // const url = parseUrl(this.page.url());
+	 // const host = '.' + url.host.split('.').slice(-2).join('.');
+	 // let cookiesVal = read(file, 'utf8');
+	 // cookiesVal = this.jsonArray(cookiesVal, host);
+	
+	 */
+	/**
+	 * 解析网页set-cookies的值
+	 *
+	 * @param {Object} strCookies
+	 */
+	async parseCookies(strCookies) {
+		return await strCookies.split("\n").map((ls) => {
+			let value = {};
+			ls.split(';').map((ln, j) => {
+				// console.log(ln);
+				const arr = ln.split('=');
+				const Key = (arr[0]).trim();
+				if (!Key) return;
+
+				if (j === 0) {
+					value.name = Key;
+					value.value = arr[1];
+				}
+				else if (Key === 'Max-Age') {
+					value.expire = parseInt(arr[1]) + (Date.now() / 1000);
+				}
+				else if (Key === 'Secure') {
+					value.source = true;
+					value.sourceScheme = 'Secure';
+				}
+				else {
+					value[Key[0].toLowerCase() + Key.substring(1)] = arr[1] || true;
+				}
+			})
+			return value;
+		})
+	}
+
+	/**
+	 * 合并两个Cookies，并以后面的值为准
+	 *
+	 * @param {Object} oldCookies
+	 * @param {Object} newCookies
+	 */
+	async mergeNewCookies(oldCookies, newCookies) {
+		if (newCookies.length === 0) return oldCookies;
+		let aMap = new Map(oldCookies.map(obj => [obj.name, obj]));
+		// let aMap = JSON.parse(JSON.stringify(this.cookies)); //深拷贝
+		newCookies.map(obj => {
+			aMap.set(obj.name, obj)
+		});
+		return await Array.from(aMap.values());
+	}
+
+
+	/**
+	 * 获取当前页面的Cookies
+	 */
+	async getCookies() {
+		try {
+			return await this.page.cookies();
+		}
+		catch (e) {
+			console.log('[chrome.getCookies.Error]', e.message);
+			return [];
+		}
+	}
+
+	/**
+	 * 设置Cookies
+	 * @param {Object} cookies
+	 */
+	async setCookies(cookies) {
+		try {
+			await this.page.setCookie(...cookies.map(ck => {
+				if (typeof ck.expires === 'string') ck.expires = new Date(ck.expires).getTime();
+				return ck;
+			}).filter(ck => (!!ck.domain || !!ck.url)));
+		}
+		catch (e) {
+			console.log('[chrome.setCookies.Error]', e.message);
+			console.log(JSON.stringify(cookies));
+		}
+	}
+
+	/**
+	 * 保存当前页面中的Cookies
+	 * @param {Object} file
+	 */
+	async saveCookies(file, append = true) {
+		try {
+			let cookies = await this.page.cookies();
+
+			if (append && fs.existsSync(file)) {
+				let dbCookies = JSON.parse(fs.readFileSync(file, 'utf8'));
+				cookies = this.mergeNewCookies(dbCookies, cookies);
+			}
+			await fs.writeFileSync(file, JSON.stringify(cookies, null, 2));
+		}
+		catch (e) {
+			console.log('[chrome.saveCookies.Error]', e.message);
+		}
+	}
+
+
+
+}