@kne/fastify-puppeteer 1.1.1 → 1.1.3

package/index.js

@@ -10,6 +10,7 @@ module.exports = fp(
         root: path.resolve(process.cwd(), '.puppeteer_cache'),
         prefix: `/api/v${packageJson.version.split('.')[0]}/puppeteer`,
         maxCacheKeys: 1000,
+        maxConcurrent: 10,
         maxTaskSize: 100,
         pageWidth: 1366,
         pageHeight: 768,

package/libs/core/file-cache.js

@@ -1,48 +1,51 @@
-const { LRUCache } = require('lru-cache');
+const {LRUCache} = require('lru-cache');
 const path = require('path');
 const fs = require('fs-extra');
 const merge = require('lodash/merge');
 const fileType = require('file-type');
 const objectHash = require('object-hash');
-const { v4: uuidv4 } = require('uuid');
+const {v4: uuidv4} = require('uuid');
+const crypto = require('crypto');
 
 module.exports = async options => {
-  options = merge(
-    {},
-    {
-      maxCacheKeys: 1000,
-      root: path.resolve(process.cwd(), '.puppeteer_cache')
-    },
-    options
-  );
+  options = merge({}, {
+    maxCacheKeys: 1000, root: path.resolve(process.cwd(), '.puppeteer_cache')
+  }, options);
   await fs.emptyDir(options.root);
   const cache = new LRUCache({
-    max: options.maxCacheKeys,
-    dispose: async value => {
+    max: options.maxCacheKeys, dispose: async value => {
       await fs.remove(path.resolve(options.root, value));
     }
   });
   console.log('init cache dir');
   return {
     setCache: async (object, file) => {
+      const hash = objectHash(object);
       if (Buffer.isBuffer(file)) {
-        const { ext } = await fileType.fromBuffer(file);
-        const filename = uuidv4() + '.' + ext;
+        const {ext} = await fileType.fromBuffer(file);
+        const filename = crypto.createHash('md5').update(file).digest('hex') + '.' + ext;
         const filePath = path.resolve(options.root, filename);
         await fs.writeFile(filePath, file);
-        cache.set(objectHash(object), filename);
+        cache.set(hash, filename);
         return filename;
       } else {
         const filename = uuidv4() + '.zip';
         const filePath = path.resolve(options.root, filename);
+        const hash = crypto.createHash('md5');
         await new Promise((resolve, reject) => {
-          file.pipe(fs.createWriteStream(filePath)).on('error', reject).on('finish', resolve);
+          file
+            .on('data', chunk => hash.update(chunk))
+            .pipe(fs.createWriteStream(filePath))
+            .on('error', reject)
+            .on('finish', resolve);
         });
-        cache.set(objectHash(object), filename);
-        return filename;
+        const newFilename = hash.digest('hex') + '.zip';
+        const newFilePath = path.resolve(options.root, newFilename);
+        await fs.move(filePath, newFilePath);
+        cache.set(hash, newFilename);
+        return newFilename;
       }
-    },
-    getCache: async object => {
+    }, getCache: async object => {
       const filename = cache.get(objectHash(object));
       if (!filename) {
         return null;

package/libs/services.js

@@ -3,12 +3,15 @@ const setup = require('./core/setup');
 const createCache = require('./core/file-cache');
 const compressing = require('compressing');
 const path = require('path');
+const createDeferred = require('@kne/create-deferred');
 
 module.exports = fp(async (fastify, options) => {
-  const { maxCacheKeys, root } = options;
+  const { maxCacheKeys, maxConcurrent, root } = options;
   const cache = await createCache(options);
   const puppeteerPage = await setup(options);
 
+  const deferred = createDeferred(maxConcurrent);
+
   const memoCache = (cacheName, callback) => async props => {
     const currentCache = await cache.getCache(Object.assign({}, props, { cacheName }));
     if (currentCache) {
@@ -27,8 +30,11 @@ module.exports = fp(async (fastify, options) => {
     }
     const zipStream = new compressing.zip.Stream();
     const files = [];
+
     for (let target of list) {
-      files.push(await callback(target));
+      await deferred(async () => {
+        files.push(await callback(target));
+      });
     }
     files.forEach(filename => {
       zipStream.addEntry(path.resolve(root, filename));
@@ -43,7 +49,7 @@ module.exports = fp(async (fastify, options) => {
         Object.assign(
           {},
           {
-            waitUntil: 'networkidle2',
+            waitUntil: 'networkidle0',
             displayHeaderFooter: true,
             printBackground: true,
             format: 'A4'
@@ -63,7 +69,7 @@ module.exports = fp(async (fastify, options) => {
   const parseHtmlToPhoto = memoCache('parseHtmlToPhoto', async ({ html, options }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.setContent(html);
-      return await page.screenshot(Object.assign({}, { waitUntil: 'networkidle2', type: 'png' }, options));
+      return await page.screenshot(Object.assign({}, { waitUntil: 'networkidle0', type: 'png' }, options));
     });
   });
 
@@ -76,7 +82,7 @@ module.exports = fp(async (fastify, options) => {
   const parseUrlToPdf = memoCache('parseUrlToPdf', async ({ url, options = {} }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.goto(url, {
-        waitUntil: 'networkidle2'
+        waitUntil: 'networkidle0'
       });
       await Promise.all(
         (options.waitForSelectors || []).map(waitForSelector => {
@@ -109,7 +115,7 @@ module.exports = fp(async (fastify, options) => {
   const parseUrlToPhoto = memoCache('parseUrlToPhoto', async ({ url, selector, options = {} }) => {
     return await puppeteerPage.task(async ({ page }) => {
       await page.goto(url, {
-        waitUntil: 'networkidle2'
+        waitUntil: 'networkidle0'
       });
       await Promise.all(
         (options.waitForSelectors || []).map(waitForSelector => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kne/fastify-puppeteer",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "用于操作无头浏览器截图保存pdf等",
   "main": "index.js",
   "scripts": {