nodejs_chromium 1.1.11 → 1.1.15

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "nodejs_chromium",
-	"version": "1.1.11",
+	"version": "1.1.15",
 	"description": "for pupeteer chromium",
 	"main": "index.js",
 	"scripts": {

package/src/chrome.js

@@ -1,6 +1,6 @@
 const fs = require("fs");
 const Cookies = require("./cookies");
-const { parse: parseUrl } = require("url");
+const { URL } = require('url');
 
 
 /**
@@ -53,7 +53,7 @@ module.exports = class {
 			await page.evaluateOnNewDocument(() => {
 				const newProto = navigator.__proto__;
 				delete newProto.webdriver; //删除 navigator.webdriver字段
-				navigator.__proto__ = newProto; //在每次新文档加载时，删除 navigator.webdriver 字段，这有助于防止某些网站检测到自动化行为。
+				navigator.__proto__ = newProto; //删除 navigator.webdriver 字段，有助于防止某些网站检测到自动化行为
 			});
 			return new module.exports(this.browser, page, this.options, false); //new 自身
 		}
@@ -152,7 +152,7 @@ module.exports = class {
 	/**
 	 * 关闭
 	 */
-	async close(act = 0) {
+	async close(act = 1) {
 		try {
 			if (act & 1) await this.page.close();
 			if (act & 2) await this.browser.close();
@@ -305,15 +305,18 @@ module.exports = class {
 	 */
 	async elements(tag1, tag2, call) {
 		try {
+
 			const div = await this.page.$(tag1);
 			if (!div) {
 				throw new Error(`${tag1} not exists`);
 			}
+			let index = 0;
 			for (const elm of (await div.$$(tag2))) {
 				const html = await elm.evaluate(node => node.outerHTML);
 				const text = await elm.evaluate(node => node.innerHTML);
-				call(elm, { html, text });
+				call(elm, { html, text, index: index++ });
 			}
+
 		}
 		catch (e) {
 			console.log('[chrome.elements.Error]', e.message)
@@ -321,6 +324,40 @@ module.exports = class {
 	}
 
 
+	/**
+	 * 遍历tag1里的tag2，并由call判断，只有call返回true时才resolve
+	 *
+	 * @param {Object} tag1
+	 * @param {Object} tag2
+	 * @param {Object} call
+	 */
+	async search(tag1, tag2, call) {
+		return await new Promise(async (resolve) => {
+			try {
+
+				const div = await this.page.$(tag1);
+				if (!div) {
+					throw new Error(`${tag1} not exists`);
+				}
+				let index = 0;
+				for (const ele of (await div.$$(tag2))) {
+					index++;
+					const html = await ele.evaluate(node => node.outerHTML);
+					const text = await ele.evaluate(node => node.innerHTML);
+					let val = call(ele, { html, text, index });
+					if (val === true) return resolve({ success: true, ele, html, text, index });
+				}
+				resolve({ success: false });
+
+			}
+			catch (e) {
+				console.log('[chrome.elements.Error]', e.message)
+			}
+		});
+
+	}
+
+
 	/**
 	 * 支持css普通选择器方式和伪类方式
 	 * div.body
@@ -444,8 +481,8 @@ module.exports = class {
 	async click(el, option = {}) {
 		try {
 			let { delay = 100, count = 1, x = 6, y = 3 } = option;
-			await this.page.click(el, { delay, count, offset: { x, y } });
-			return this;
+			return await this.page.click(el, { delay, count, offset: { x, y } });
+			// return this;
 		}
 		catch (e) {
 			console.log('[chrome.click.Error]', e.message);
@@ -577,7 +614,7 @@ module.exports = class {
 	 * 补全所有本地js/css，一般用于保存html之前
 	 */
 	async improveUrls() {
-		const url = parseUrl(this.page.url());
+		const url = new URL(this.page.url());
 		const domain = url.protocol + '//' + url.host;
 		await this.page.evaluate((domain) => {
 			try {
@@ -595,6 +632,18 @@ module.exports = class {
 		}, domain);
 	}
 
+	skipTypes = ['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping'];
+
+	setSkipType(items) {
+		this.skipTypes.length = 0;
+		this.skipTypes = [...items];
+		return this;
+	}
+
+	delSkipType(...item) {
+		this.skipTypes = this.skipTypes.del(...item);
+		return this;
+	}
 
 	async parseResponse(response) {
 		// const response = await this.page.waitForResponse(res => res);
@@ -611,7 +660,7 @@ module.exports = class {
 		if (value.type === 'xhr') value.type = 'AJAX';
 		value.url = await response.url();
 		value.headers = { request: request.headers(), response: headers };
-		value.domain = parseUrl(value.url)['host'];
+		value.domain = (new URL(value.url))['host'];
 		value.content = headers['content-type'];
 		value.length = headers['content-length'];
 		value.status = await response.status();
@@ -622,7 +671,11 @@ module.exports = class {
 		if (headers['set-cookie']) value.cookies = await this.cookies.parse(headers['set-cookie']);
 		value.remote = await response.remoteAddress(); //目标服务器
 		if (value.status === 301 || value.status === 302) return value;
-		if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping'].has(value.type)) return value;
+
+		// if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping'].has(value.type)) return value;
+		// if (['image', 'font', 'other', 'script', 'stylesheet', 'ping'].includes(value.type)) return value;
+		if (this.skipTypes.includes(value.type)) return value;
+
 		if (value.content) {
 			if (value.content.startsWith('application/vnd')) return value;
 			if (value.content.startsWith('application/xml')) return value;
@@ -646,9 +699,8 @@ module.exports = class {
 	}
 
 	host(url) {
-		// const domain = url.protocol + '//' + url.host;
-		const urls = parseUrl(url);
-		return '.' + urls.host.split('.').slice(-2).join('.');
+		const myUrl = new URL(url);
+		return '.' + myUrl.host.split('.').slice(-2).join('.');
 	}
 
 
@@ -669,7 +721,7 @@ module.exports = class {
 
 			if (abort.length > 0 && abort.some(t => t === rType)) return request.abort();
 
-			if (optHead !== {}) Object.assign(headers, optHead);
+			if (JSON.stringify(optHead) !== '{}') Object.assign(headers, optHead);
 
 			headers['Access-Control-Allow-Origin'] = '*'; // 设置允许跨源访问的域名，可以根据需求修改
 			headers['Access-Control-Allow-Methods'] = '*'; //'GET, POST, PUT, OPTIONS';
@@ -679,8 +731,7 @@ module.exports = class {
 
 		this.page.on('response', async res => {
 			if (!this.responseCall) return;
-			let json = await this.parseResponse(res);
-			await this.responseCall(json);
+			await this.responseCall(await this.parseResponse(res));
 		});
 
 	}