@kne/fastify-puppeteer 1.1.1 → 1.1.2

package/libs/core/file-cache.js

@@ -5,6 +5,7 @@ const merge = require('lodash/merge');
 const fileType = require('file-type');
 const objectHash = require('object-hash');
 const { v4: uuidv4 } = require('uuid');
+const crypto = require('crypto');
 
 module.exports = async options => {
   options = merge(
@@ -25,20 +26,23 @@ module.exports = async options => {
   console.log('init cache dir');
   return {
     setCache: async (object, file) => {
+      const hash = objectHash(object);
+      const md5 = crypto.createHash('md5').update(file).digest('hex');
       if (Buffer.isBuffer(file)) {
         const { ext } = await fileType.fromBuffer(file);
-        const filename = uuidv4() + '.' + ext;
+        const filename = md5 + '.' + ext;
         const filePath = path.resolve(options.root, filename);
         await fs.writeFile(filePath, file);
-        cache.set(objectHash(object), filename);
+        cache.set(hash, filename);
         return filename;
       } else {
-        const filename = uuidv4() + '.zip';
+        const filename = md5 + '.zip';
         const filePath = path.resolve(options.root, filename);
         await new Promise((resolve, reject) => {
           file.pipe(fs.createWriteStream(filePath)).on('error', reject).on('finish', resolve);
         });
-        cache.set(objectHash(object), filename);
+
+        cache.set(hash, filename);
         return filename;
       }
     },

package/libs/services.js

@@ -3,12 +3,15 @@ const setup = require('./core/setup');
 const createCache = require('./core/file-cache');
 const compressing = require('compressing');
 const path = require('path');
+const createDeferred = require('@kne/create-deferred');
 
 module.exports = fp(async (fastify, options) => {
-  const { maxCacheKeys, root } = options;
+  const { maxCacheKeys, maxConcurrency = 10, root } = options;
   const cache = await createCache(options);
   const puppeteerPage = await setup(options);
 
+  const deferred = createDeferred(maxConcurrency);
+
   const memoCache = (cacheName, callback) => async props => {
     const currentCache = await cache.getCache(Object.assign({}, props, { cacheName }));
     if (currentCache) {
@@ -27,8 +30,11 @@ module.exports = fp(async (fastify, options) => {
     }
     const zipStream = new compressing.zip.Stream();
     const files = [];
+
     for (let target of list) {
-      files.push(await callback(target));
+      await deferred(async () => {
+        files.push(await callback(target));
+      });
     }
     files.forEach(filename => {
       zipStream.addEntry(path.resolve(root, filename));
@@ -43,7 +49,7 @@ module.exports = fp(async (fastify, options) => {
         Object.assign(
           {},
           {
-            waitUntil: 'networkidle2',
+            waitUntil: 'networkidle0',
             displayHeaderFooter: true,
             printBackground: true,
             format: 'A4'
@@ -63,7 +69,7 @@ module.exports = fp(async (fastify, options) => {
   const parseHtmlToPhoto = memoCache('parseHtmlToPhoto', async ({ html, options }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.setContent(html);
-      return await page.screenshot(Object.assign({}, { waitUntil: 'networkidle2', type: 'png' }, options));
+      return await page.screenshot(Object.assign({}, { waitUntil: 'networkidle0', type: 'png' }, options));
     });
   });
 
@@ -76,7 +82,7 @@ module.exports = fp(async (fastify, options) => {
   const parseUrlToPdf = memoCache('parseUrlToPdf', async ({ url, options = {} }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.goto(url, {
-        waitUntil: 'networkidle2'
+        waitUntil: 'networkidle0'
       });
       await Promise.all(
         (options.waitForSelectors || []).map(waitForSelector => {
@@ -109,7 +115,7 @@ module.exports = fp(async (fastify, options) => {
   const parseUrlToPhoto = memoCache('parseUrlToPhoto', async ({ url, selector, options = {} }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.goto(url, {
-        waitUntil: 'networkidle2'
+        waitUntil: 'networkidle0'
       });
       await Promise.all(
         (options.waitForSelectors || []).map(waitForSelector => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kne/fastify-puppeteer",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "用于操作无头浏览器截图保存pdf等",
   "main": "index.js",
   "scripts": {