nodejs_chromium 1.1.13 → 1.1.16

package/index.js

@@ -1,5 +1,6 @@
 require('nodejs_patch');
 const puppeteer = require("puppeteer");
+const { KnownDevices } = require("puppeteer");
 const chrome = require("./src/chrome.js");
 const { exec } = require("child_process");
 global.__UA__ = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36';
@@ -16,13 +17,17 @@ async function NormalClose(path) {
 	 */
 	let file = `${path}/Default/Preferences`;
 	// console.log({file});
-	if (!fs.existsSync(file)) return;
+	if (!fs.existsSync(file)) {
+		console.log(file, 'NotExists')
+		return;
+	}
 	let text = fs.readFileSync(file);
 	let json = JSON.parse(text);
 	json.credentials_enable_service = false; //新增项，禁止保存密码
 	json.profile.exit_type = 'Normal'; //原值可能为：Crashed
 	// console.log('json.profile.exit_type', json.profile.exit_type);
 	fs.writeFileSync(file, JSON.stringify(json));
+	console.log(file, json.profile.exit_type)
 }
 
 
@@ -99,7 +104,7 @@ async function newBrowser(params) {
 
 		const browser = await puppeteer.launch(option);
 
-		await NormalClose(option.userDataDir); //禁止显示【要恢复页面吗？chromium 未正确关闭。】
+		// await NormalClose(option.userDataDir); //禁止显示【要恢复页面吗？chromium 未正确关闭。】
 
 		return browser;
 	}
@@ -126,6 +131,7 @@ async function newChrome(params) {
 				cookies = null, //若cookies=false，则不处理cookies，不指定则由chrome处理，若=文件路径
 				abort = [], //要禁止的类型，常见["font","image","ping","stylesheet","document","fetch","script","xhr"]
 				headers = {},
+				device = null
 		} = params;
 
 		const browser = await newBrowser(params);
@@ -136,11 +142,14 @@ async function newChrome(params) {
 		// const version = await browser.version();
 		const page = (await browser.pages())[0];
 		// const page = await browser.newPage();
-		await page.setRequestInterception(true); //允许拦截
+		// await page.setRequestInterception(true); //允许拦截
 
 		if (ua) await page.setUserAgent(ua);
-		else if (mobile) await page.setUserAgent(mob_ua);
+		else if (mobile) {
+			await page.emulate(device || KnownDevices['iPhone 15 Pro']);
+		}
 		else await page.setUserAgent(__UA__);
+		if (device) await page.emulate(device); //必须要放在setUserAgent后面
 
 		await page.evaluateOnNewDocument(() => {
 			const newProto = navigator.__proto__;
@@ -149,7 +158,7 @@ async function newChrome(params) {
 		});
 
 		if (cookies !== false) cookies = `runtime/.cache/${id}/cookies.json`;
-		const pageOption = { cookies, visible, abort, headers };
+		const pageOption = { cookies, visible, abort, headers, device };
 		return new chrome(browser, page, pageOption);
 	}
 	catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "nodejs_chromium",
-	"version": "1.1.13",
+	"version": "1.1.16",
 	"description": "for pupeteer chromium",
 	"main": "index.js",
 	"scripts": {

package/src/chrome.js

@@ -2,7 +2,6 @@ const fs = require("fs");
 const Cookies = require("./cookies");
 const { URL } = require('url');
 
-
 /**
  *
  * class CdpFrame extends _classSuper
@@ -19,8 +18,10 @@ module.exports = class {
 	requestCall = void 0;
 	cookies = void 0;
 	options = void 0;
+	device = void 0;
 	isFrame = false; //是不是在iFrame中
 	visible = false; //是否可见，也就是有没有启动窗口
+	resIndex = 0; //当前窗口经过response的序号
 
 	constructor(browser, page, options, isFrame = false) {
 		this.browser = browser;
@@ -28,28 +29,78 @@ module.exports = class {
 		this.options = options;
 		this.cookies = new Cookies(page, options.cookies);
 		this.visible = !!options.visible;
+		this.device = options.device;
 		this.isFrame = !!isFrame;
 		this.doListening(options);
 	}
 
-	request(call) {
+	/**
+	 * 设置拦截
+	 * 如果只是要读取cookies这些信息而不拦截处理，只需调用interception
+	 *
+	 * @param call
+	 * @returns {Promise<exports>}
+	 */
+	async request(call) {
+
+		if (call === true) {
+			//等同于this.interception(true)
+			await this.page.setRequestInterception(true); //允许拦截
+			this.page.interception = true;
+			this.requestCall = null;
+			return this;
+
+		}
+		else if (call === false) {
+			//等同于this.interception(false)
+			await this.page.setRequestInterception(false); //关闭允许拦截
+			this.page.interception = false;
+			this.requestCall = null;
+			return this;
+
+		}
+
+		await this.page.setRequestInterception(true); //允许拦截
+		this.page.interception = true;
 		this.requestCall = call;
 		return this;
 	}
 
+
+	/**
+	 * 设置是否允许拦截
+	 *
+	 * @param val
+	 * @returns {Promise<exports>}
+	 */
+	async interception(val = true) {
+		await this.page.setRequestInterception(val);
+		this.page.interception = val;
+		return this;
+	}
+
+
 	response(call) {
 		this.responseCall = call;
 		return this;
 	}
 
+	newPage(device = null) {
+		return this.clone(device);
+	}
+
 	/**
 	 * 重新在当前浏览器创建新窗口
 	 */
-	async clone() {
+	async clone(device = null) {
 		try {
 			const page = await this.browser.newPage();
 			await page.setBypassCSP(true); //绕过页面的内容安全策略
-			await page.setRequestInterception(true); //允许拦截
+			// await page.setRequestInterception(true); //允许拦截
+
+			if (device) await page.emulate(device);
+			else if (this.options.device) await page.emulate(this.options.device);
+
 			await page.evaluateOnNewDocument(() => {
 				const newProto = navigator.__proto__;
 				delete newProto.webdriver; //删除 navigator.webdriver字段
@@ -108,10 +159,10 @@ module.exports = class {
 	/**
 	 * 这必须放在已经打开过这个URL之后
 	 */
-	waitForFrame() {
+	waitForFrame(name = 'Test') {
 		try {
 			this.page.waitForFrame(async frame => {
-				return frame.name() === 'Test';
+				return await frame.name() === name;
 			});
 			return this;
 		}
@@ -127,7 +178,7 @@ module.exports = class {
 	async waitFrame(urlKey, timeout = 5000) {
 		try {
 			return await this.page.waitForFrame(async frame => {
-				return frame.url().indexOf(urlKey) > 0;
+				return frame.url().includes(urlKey);
 			}, { timeout });
 		}
 		catch (e) {
@@ -137,6 +188,7 @@ module.exports = class {
 
 	/**
 	 * 按tag创建当前窗口中的某个iFrame
+	 * 
 	 * @param {Object} tag
 	 */
 	async iframe(tag) {
@@ -245,6 +297,21 @@ module.exports = class {
 		}
 	}
 
+
+	/**
+	 * 获取部分或全部HTML
+	 * @param {Object} obj
+	 */
+	async evaluate(obj, callback) {
+		try {
+			return await this.page.evaluate(callback, obj);
+		}
+		catch (e) {
+			console.log('[chrome.evaluate.Error]', e.message);
+		}
+	}
+
+
 	/**
 	 * 获取部分或全部HTML
 	 * @param {Object} obj
@@ -291,7 +358,7 @@ module.exports = class {
 			return this;
 		}
 		catch (e) {
-			console.log('[chrome.text.Error]', e.message);
+			console.log('[chrome.display.Error]', [tag, e.message]);
 		}
 	}
 
@@ -305,22 +372,56 @@ module.exports = class {
 	 */
 	async elements(tag1, tag2, call) {
 		try {
-			const div = await this.page.$(tag1);
-			if (!div) {
-				throw new Error(`${tag1} not exists`);
-			}
-			for (const elm of (await div.$$(tag2))) {
-				const html = await elm.evaluate(node => node.outerHTML);
-				const text = await elm.evaluate(node => node.innerHTML);
-				call(elm, { html, text });
+			let has = await this.wait(tag, 2, 2);
+			if (!has) return null;
+
+			let index = 0;
+			for (const ele of (await div.$$(tag2))) {
+				const html = await ele.evaluate(node => node.innerHTML);
+				const text = await ele.evaluate(node => node.innerText);
+				await call(ele, { html, text, index: index++ });
 			}
+
 		}
 		catch (e) {
-			console.log('[chrome.elements.Error]', e.message)
+			console.log('[chrome.elements.Error]', [tag1, tag2, e.message])
 		}
 	}
 
 
+	/**
+	 * 遍历tag1里的tag2，并由call判断，只有call返回true时才resolve
+	 *
+	 * @param {Object} tag1
+	 * @param {Object} tag2
+	 * @param {Object} call
+	 */
+	async search(tag1, tag2, call) {
+		return await new Promise(async (resolve) => {
+			try {
+				let has = await this.wait(tag, 2, 2);
+				if (!has) return {};
+
+				let index = 0;
+				for (const ele of (await div.$$(tag2))) {
+					index++;
+					const html = await ele.evaluate(node => node.innerHTML);
+					const text = await ele.evaluate(node => node.innerText);
+					let val = await call(ele, { html, text, index });
+					if (val === true) return resolve({ success: true, ele, html, text, index });
+				}
+				resolve({ success: false, index });
+
+			}
+			catch (e) {
+				console.log('[chrome.search.Error]', [tag1, tag2, e.message])
+				return {};
+			}
+		});
+
+	}
+
+
 	/**
 	 * 支持css普通选择器方式和伪类方式
 	 * div.body
@@ -336,11 +437,14 @@ module.exports = class {
 	 */
 	async element(tag, selector = false) {
 		try {
+			let has = await this.wait(tag, 2, 2);
+			if (!has) return null;
+
 			if (selector) return await this.page.querySelector(tag);
 			return await this.page.$(tag);
 		}
 		catch (e) {
-			console.log('[chrome.element.Error]', e.message);
+			console.log('[chrome.element.Error]', [tag, e.message]);
 			return null;
 		}
 	}
@@ -400,9 +504,6 @@ module.exports = class {
 		}
 	}
 
-	/**
-	 * delay=每键入一个字符延迟毫秒
-	 */
 	async clear(el) {
 		try {
 			await this.page.$eval(el, ele => ele.value = '');
@@ -443,12 +544,12 @@ module.exports = class {
 	 */
 	async click(el, option = {}) {
 		try {
+
 			let { delay = 100, count = 1, x = 6, y = 3 } = option;
-			await this.page.click(el, { delay, count, offset: { x, y } });
-			return this;
+			return await this.page.click(el, { delay, count, offset: { x, y } });
 		}
 		catch (e) {
-			console.log('[chrome.click.Error]', e.message);
+			console.log('[chrome.click.Error]', el, e.message);
 		}
 	}
 
@@ -596,65 +697,91 @@ module.exports = class {
 	}
 
 	skipTypes = ['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping'];
+
 	setSkipType(items) {
 		this.skipTypes.length = 0;
 		this.skipTypes = [...items];
 		return this;
 	}
 
-	delSkipType(...item) {
+	delSkipType(item) {
 		this.skipTypes = this.skipTypes.del(...item);
 		return this;
 	}
 
+
 	async parseResponse(response) {
 		// const response = await this.page.waitForResponse(res => res);
-
 		const value = {};
-		const request = await response.request();
-		const headers = await response.headers();
-		value.method = await request.method();
-		if (value.method === 'OPTIONS') return;
-
-		value.type = await request.resourceType();
-		// value.redirect = await response.redirectURL();
-		if (value.type === 'fetch') value.type = 'xhr';
-		if (value.type === 'xhr') value.type = 'AJAX';
-		value.url = await response.url();
-		value.headers = { request: request.headers(), response: headers };
-		value.domain = (new URL(value.url))['host'];
-		value.content = headers['content-type'];
-		value.length = headers['content-length'];
-		value.status = await response.status();
-		value.ok = await response.ok();
-		value.datetime = (new Date(headers['date'])).date('yyyy-mm-dd hh:ii:ss');
-		// value.headers = headers;
-		if (headers['server']) value.server = headers['server'];
-		if (headers['set-cookie']) value.cookies = await this.cookies.parse(headers['set-cookie']);
-		value.remote = await response.remoteAddress(); //目标服务器
-		if (value.status === 301 || value.status === 302) return value;
-
-		// if (['image', 'font', 'other', 'script', 'stylesheet', 'document', 'ping'].has(value.type)) return value;
-		// if (['image', 'font', 'other', 'script', 'stylesheet', 'ping'].includes(value.type)) return value;
-		if (this.skipTypes.includes(value.type)) return value;
-
-		if (value.content) {
-			if (value.content.startsWith('application/vnd')) return value;
-			if (value.content.startsWith('application/xml')) return value;
-			if (value.content.startsWith('text/css')) return value;
-		}
-
-		value.post = await request.postData();
-		if (value.post) value.post = value.post.toString();
+
+		try {
+
+			const request = await response.request();
+			const headers = await response.headers();
+
+			value.method = await request.method();
+			if (value.method === 'OPTIONS') return;
+			// if (value.method === 'DELETE') return;
+			value.index = this.resIndex++
+
+			value.type = await request.resourceType();
+			// value.redirect = await response.redirectURL();
+			if (value.type === 'fetch') value.type = 'xhr';
+			if (value.type === 'xhr') value.type = 'AJAX';
+			value.url = await response.url();
+			value.headers = { request: request.headers(), response: headers };
+			value.domain = (new URL(value.url))['host'];
+			value.content = headers['content-type'];
+			value.length = headers['content-length'];
+			value.status = await response.status();
+			value.ok = await response.ok();
+			value.datetime = (new Date(headers['date'])).date('yyyy-mm-dd hh:ii:ss');
+			if (headers['server']) value.server = headers['server'];
+			if (headers['set-cookie']) value.cookies = await this.cookies.parse(headers['set-cookie']);
+			value.remote = await response.remoteAddress(); //目标服务器
+			if (value.status === 301 || value.status === 302) return value;
+			if (this.skipTypes.includes(value.type)) return value;
+
+			value.post = await request.postData();
+			if (value.post) value.post = value.post.toString();
+
+		}
+		catch (e) {
+			const err = e.parse();
+			value.error = err.message;
+		}
+
+		try {
+
+			if (value.content.includes('text/html')) {
+				const buffer = await response.buffer();
+				value.html = buffer.toString();
+				return value;
+			}
+
+			if (value.content) {
+				if (value.content.includes('application/vnd')) return value;
+				if (value.content.includes('application/xml')) return value;
+				if (value.content.includes('application/javascript')) return value;
+				if (value.content.includes('text/css')) return value;
+				if (value.content.includes('text/plain')) return value;
+			}
+
+		}
+		catch (e) {
+			const err = e.parse();
+			value.error = err.message;
+		}
 
 		try {
 			const buffer = await response.buffer();
 			value.response = buffer.toString();
+			if (!/^[\[\{].+[\]\}]$/.test(value.response)) return value;
 			value.json = JSON.parse(value.response);
 		}
 		catch (e) {
 			const err = e.parse();
-			value.json = err.message;
+			value.error = err.message;
 		}
 
 		return value;
@@ -669,6 +796,8 @@ module.exports = class {
 	async doListening(options) {
 
 		this.page.on('request', async (request) => {
+			if (!this.page.interception) return;
+
 			const host = this.host(request.url());
 			const headers = request.headers();
 			if (headers.cookie) this.cookies.request(headers.cookie, host);
@@ -676,6 +805,7 @@ module.exports = class {
 			if (this.requestCall) {
 				const run = await this.requestCall(request);
 				if (run === false) return request.abort();
+				if (run === true) return;
 			}
 
 			const rType = request.resourceType();