@kne/fastify-puppeteer 1.0.0

package/README.md

@@ -0,0 +1,27 @@
+
+# fastify-puppeteer
+
+
+### 描述
+
+用于操作无头浏览器截图保存pdf等
+
+
+### 安装
+
+```shell
+npm i --save @kne/fastify-puppeteer
+```
+
+### 示例
+
+#### 示例代码
+
+
+
+### API
+
+| 属性名 | 说明 | 类型 | 默认值 |
+|-----|----|----|-----|
+|     |    |    |     |
+

package/index.js

@@ -0,0 +1,41 @@
+const fp = require('fastify-plugin');
+const path = require('path');
+const packageJson = require('./package.json');
+const merge = require('lodash/merge');
+
+module.exports = fp(
+  async (fastify, options) => {
+    options = merge(
+      {
+        root: path.resolve(process.cwd(), '.puppeteer_cache'),
+        prefix: `/api/v${packageJson.version.split('.')[0]}/puppeteer`,
+        maxCacheKeys: 1000,
+        maxTaskSize: 100,
+        puppeteerOptions: {},
+        authenticate: []
+      },
+      options
+    );
+
+    fastify.register(
+      require('@fastify/static'),
+      Object.assign({}, options.static, {
+        root: options.root,
+        index: false,
+        list: false
+      })
+    );
+
+    fastify.register(require('@kne/fastify-namespace'), {
+      name: 'puppeteer',
+      options,
+      modules: [
+        ['services', path.resolve(__dirname, './libs/services.js')],
+        ['controllers', path.resolve(__dirname, './libs/controllers.js')]
+      ]
+    });
+  },
+  {
+    name: 'fastify-puppeteer'
+  }
+);

package/libs/controllers.js

@@ -0,0 +1,219 @@
+const fp = require('fastify-plugin');
+module.exports = fp(async (fastify, options) => {
+  const { services } = fastify.puppeteer;
+  fastify.post(
+    `${options.prefix}/parseHtmlToPdf`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'html生成pdf文件流',
+        body: {
+          type: 'object',
+          required: ['content'],
+          properties: {
+            options: { type: 'object' },
+            content: { type: 'string' }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseHtmlToPdf({ html: request.body.content, options: request.body.options });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseHtmlToPdfBatch`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: '批量将html生成pdf的zip文件流',
+        body: {
+          type: 'object',
+          required: ['contentList'],
+          properties: {
+            options: { type: 'object' },
+            contentList: {
+              type: 'array',
+              items: {
+                type: 'string'
+              }
+            }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseHtmlToPdfBatch({
+        htmlList: request.body.contentList,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseHtmlToPhotoBatch`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: '批量将html生成png的zip文件流',
+        body: {
+          type: 'object',
+          required: ['contentList'],
+          properties: {
+            options: { type: 'object' },
+            contentList: {
+              type: 'array',
+              items: {
+                type: 'string'
+              }
+            }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseHtmlToPhotoBatch({
+        htmlList: request.body.contentList,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseHtmlToPhoto`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'html生成png文件流',
+        body: {
+          type: 'object',
+          required: ['content'],
+          properties: {
+            options: { type: 'object' },
+            content: { type: 'string' }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseHtmlToPhoto({
+        html: request.body.content,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseUrlToPdf`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'url生成pdf文件流',
+        body: {
+          type: 'object',
+          required: ['url'],
+          properties: {
+            options: { type: 'object' },
+            url: { type: 'string' }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseUrlToPdf({ url: request.body.url, options: request.body.options });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseUrlToPdfBatch`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'url生成pdf文件流',
+        body: {
+          type: 'object',
+          required: ['urlList'],
+          properties: {
+            options: { type: 'object' },
+            urlList: { type: 'array', items: { type: 'string' } }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseUrlToPdfBatch({
+        urlList: request.body.urlList,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseUrlToPhoto`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'url生成png文件流',
+        body: {
+          type: 'object',
+          required: ['url'],
+          properties: {
+            options: { type: 'object' },
+            url: { type: 'string' },
+            selector: { type: 'string' }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const file = await services.parseUrlToPhoto({
+        url: request.body.url,
+        selector: request.body.selector,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/parseUrlToPhotoBatch`,
+    {
+      onRequest: options.authenticate,
+      schema: {
+        description: '接口说明',
+        summary: 'url生成png文件流',
+        body: {
+          type: 'object',
+          required: ['urlList'],
+          properties: {
+            options: { type: 'object' },
+            urlList: { type: 'array', items: { type: 'string' } },
+            selector: { type: 'string' }
+          }
+        }
+      }
+    },
+    async (request, reply) => {
+      const filename = await services.parseUrlToPhotoBatch({
+        urlList: request.body.url,
+        selector: request.body.selector,
+        options: request.body.options
+      });
+      return reply.sendFile(filename);
+    }
+  );
+});

package/libs/core/createDeferred.js

@@ -0,0 +1,30 @@
+const createDeferred = size => {
+  const queue = [];
+  let runningCount = 0;
+  const checkRun = () => {
+    if (queue.length === 0) {
+      runningCount = 0;
+      return;
+    }
+    if (runningCount < size) {
+      const callback = queue.shift();
+      runningCount += 1;
+      callback().finally(() => {
+        runningCount -= 1;
+        checkRun();
+      });
+    }
+  };
+  return callback => {
+    return new Promise(resolve => {
+      queue.push(() => {
+        const promise = Promise.resolve().then(() => callback());
+        resolve(promise);
+        return promise;
+      });
+      checkRun();
+    });
+  };
+};
+
+module.exports = createDeferred;

package/libs/core/file-cache.js

@@ -0,0 +1,57 @@
+const { LRUCache } = require('lru-cache');
+const path = require('path');
+const fs = require('fs-extra');
+const merge = require('lodash/merge');
+const fileType = require('file-type');
+const objectHash = require('object-hash');
+const { v4: uuidv4 } = require('uuid');
+
+module.exports = async options => {
+  options = merge(
+    {},
+    {
+      maxCacheKeys: 1000,
+      root: path.resolve(process.cwd(), '.puppeteer_cache')
+    },
+    options
+  );
+  await fs.emptyDir(options.root);
+  const cache = new LRUCache({
+    max: options.maxCacheKeys,
+    dispose: async value => {
+      await fs.remove(path.resolve(options.root, value));
+    }
+  });
+  console.log('init cache dir');
+  return {
+    setCache: async (object, file) => {
+      if (Buffer.isBuffer(file)) {
+        const { ext } = await fileType.fromBuffer(file);
+        const filename = uuidv4() + '.' + ext;
+        const filePath = path.resolve(options.root, filename);
+        await fs.writeFile(filePath, file);
+        cache.set(objectHash(object), filename);
+        return filename;
+      } else {
+        const filename = uuidv4() + '.zip';
+        const filePath = path.resolve(options.root, filename);
+        await new Promise((resolve, reject) => {
+          file.pipe(fs.createWriteStream(filePath)).on('error', reject).on('finish', resolve);
+        });
+        cache.set(objectHash(object), filename);
+        return filename;
+      }
+    },
+    getCache: async object => {
+      const filename = cache.get(objectHash(object));
+      if (!filename) {
+        return null;
+      }
+      if (!(await fs.exists(path.resolve(options.root, filename)))) {
+        return null;
+      }
+
+      return filename;
+    }
+  };
+};

package/libs/core/setup.js

@@ -0,0 +1,35 @@
+const puppeteer = require('puppeteer');
+const createDeferred = require('./createDeferred');
+
+module.exports = async options => {
+  options = Object.assign({ maxTaskSize: 100 }, options);
+  const browser = await puppeteer.launch(options.puppeteerOptions);
+  const deferred = createDeferred(options.maxTaskSize);
+  return {
+    task: async callback => {
+      return await deferred(async () => {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1366, height: 768 });
+        const output = await (async () => {
+          try {
+            return {
+              success: true,
+              result: await callback({ page })
+            };
+          } catch (e) {
+            console.error(e);
+            return { success: false, error: e };
+          }
+        })();
+
+        await page.close();
+        console.log('puppeteer task complete');
+        return output;
+      });
+    },
+    close: async () => {
+      console.log('puppeteer browser close');
+      await browser.close();
+    }
+  };
+};

package/libs/core/setup.test.js

@@ -0,0 +1,17 @@
+const setup = require('./setup');
+(async () => {
+  const puppeteerPage = await setup({
+    puppeteerOptions: {
+      executablePath: 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'
+    }
+  });
+
+  const output = await puppeteerPage.task(async ({ page }) => {
+    await page.setContent('<h1>哈哈哈</h1>');
+    return await page.pdf();
+  });
+
+  console.log(output);
+
+  await puppeteerPage.close();
+})();

package/libs/services.js

@@ -0,0 +1,138 @@
+const fp = require('fastify-plugin');
+const setup = require('./core/setup');
+const createCache = require('./core/file-cache');
+const compressing = require('compressing');
+const path = require('path');
+
+module.exports = fp(async (fastify, options) => {
+  const { maxCacheKeys, root } = options;
+  const cache = await createCache(options);
+  const puppeteerPage = await setup(options);
+
+  const memoCache = (cacheName, callback) => async props => {
+    const currentCache = await cache.getCache(Object.assign({}, props, { cacheName }));
+    if (currentCache) {
+      return currentCache;
+    }
+    const output = await callback(props);
+    if (!output.success) {
+      throw output.error;
+    }
+    return await cache.setCache(props, output.result);
+  };
+
+  const batchTask = async (list, callback) => {
+    if (list.length > maxCacheKeys) {
+      throw new Error('批量操作数量超过上限');
+    }
+    const zipStream = new compressing.zip.Stream();
+    const files = [];
+    for (let target of list) {
+      files.push(await callback(target));
+    }
+    files.forEach(filename => {
+      zipStream.addEntry(path.resolve(root, filename));
+    });
+    return { result: zipStream, success: true };
+  };
+
+  const parseHtmlToPdf = memoCache('parseHtmlToPdf', async ({ html, options }) => {
+    return await puppeteerPage.task(async ({ page }) => {
+      await page.setContent(html);
+      return await page.pdf(
+        Object.assign(
+          {},
+          {
+            waitUntil: 'networkidle2',
+            displayHeaderFooter: true,
+            printBackground: true,
+            format: 'A4'
+          },
+          options
+        )
+      );
+    });
+  });
+
+  const parseHtmlToPdfBatch = memoCache('parseHtmlToPdfBatch', async ({ htmlList, options }) => {
+    return await batchTask(htmlList, async target => {
+      return await parseHtmlToPdf({ html: target, options });
+    });
+  });
+
+  const parseHtmlToPhoto = memoCache('parseHtmlToPhoto', async ({ html, options }) => {
+    return await puppeteerPage.task(async ({ page }) => {
+      await page.setContent(html);
+      return await page.screenshot(Object.assign({}, { waitUntil: 'networkidle2', type: 'png' }, options));
+    });
+  });
+
+  const parseHtmlToPhotoBatch = memoCache('parseHtmlToPhotoBatch', async ({ htmlList, options }) => {
+    return await batchTask(htmlList, async target => {
+      return await parseHtmlToPhoto({ html: target, options });
+    });
+  });
+
+  const parseUrlToPdf = memoCache('parseUrlToPdf', async ({ url, options }) => {
+    return await puppeteerPage.task(async ({ page }) => {
+      await page.goto(url, {
+        waitUntil: 'networkidle2'
+      });
+      return await page.pdf(
+        Object.assign(
+          {},
+          {
+            displayHeaderFooter: true,
+            printBackground: true,
+            format: 'A4'
+          },
+          options
+        )
+      );
+    });
+  });
+
+  const parseUrlToPdfBatch = memoCache('parseUrlToPdfBatch', async ({ urlList, options }) => {
+    return await batchTask(urlList, async target => {
+      return await parseUrlToPdf({ url: target, options });
+    });
+  });
+
+  const parseUrlToPhoto = memoCache('parseUrlToPhoto', async ({ url, selector, options }) => {
+    return await puppeteerPage.task(async ({ page }) => {
+      await page.goto(url, {
+        waitUntil: 'networkidle2'
+      });
+      if (selector) {
+        await page.locator(selector).wait();
+        const element = await page.$(selector);
+        const boundingBox = await element.boundingBox();
+        return await page.screenshot(Object.assign({}, { type: 'png', clip: boundingBox }, options));
+      }
+      return await page.screenshot(Object.assign({}, { type: 'png' }, options));
+    });
+  });
+
+  const parseUrlToPhotoBatch = memoCache('parseUrlToPhotoBatch', async ({ urlList, selector, options }) => {
+    return await batchTask(urlList, async target => {
+      return await parseUrlToPdf({ url: target, selector, options });
+    });
+  });
+
+  const getFilePath = filename => {
+    return path.resolve(root, filename);
+  };
+
+  Object.assign(fastify.puppeteer.services, {
+    puppeteerPage,
+    getFilePath,
+    parseHtmlToPdf,
+    parseHtmlToPdfBatch,
+    parseUrlToPdf,
+    parseUrlToPdfBatch,
+    parseHtmlToPhoto,
+    parseHtmlToPhotoBatch,
+    parseUrlToPhoto,
+    parseUrlToPhotoBatch
+  });
+});

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@kne/fastify-puppeteer",
+  "version": "1.0.0",
+  "description": "用于操作无头浏览器截图保存pdf等",
+  "main": "index.js",
+  "scripts": {
+    "init": "husky",
+    "dev": "nodemon ./tests/server.js",
+    "start": "node ./index.js",
+    "doc": "node ./open-api.js",
+    "build:md": "npx @kne/md-doc",
+    "start:md": "npx @kne/md-doc --watch",
+    "prettier": "prettier --config .prettierrc --write '{libs/**/*,index}.{js,jsx,ts,tsx,json,css,scss}'",
+    "lint-staged": "npx lint-staged"
+  },
+  "lint-staged": {
+    "**/*.md": [
+      "git add"
+    ],
+    "{libs/**/*,index}.{js,jsx,ts,tsx,json,css,scss}": [
+      "prettier --config .prettierrc --write",
+      "git add"
+    ]
+  },
+  "files": [
+    "index.js",
+    "libs"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kne-union/fastify-puppeteer.git"
+  },
+  "keywords": [],
+  "author": "linzp",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/kne-union/fastify-puppeteer/issues"
+  },
+  "homepage": "https://github.com/kne-union/fastify-puppeteer#readme",
+  "dependencies": {
+    "@fastify/static": "^7.0.4",
+    "@kne/fastify-namespace": "^0.1.1",
+    "compressing": "^1.10.1",
+    "fastify": "^4.27.0",
+    "fastify-plugin": "^4.5.1",
+    "file-type": "^16.5.4",
+    "fs-extra": "^11.2.0",
+    "lodash": "^4.17.21",
+    "lru-cache": "^11.0.0",
+    "object-hash": "^3.0.0",
+    "puppeteer": "^22.13.1",
+    "puppeteer-core": "^22.13.1",
+    "uuid": "^10.0.0"
+  },
+  "devDependencies": {
+    "@fastify/swagger": "^8.14.0",
+    "husky": "^9.0.11",
+    "nodemon": "^3.1.3",
+    "prettier": "^3.2.5",
+    "qs": "^6.12.1",
+    "sqlite3": "^5.1.7"
+  }
+}