@rsdoctor/utils 0.3.6 → 0.3.8-beta.0

package/dist/cjs/build/file/sharding.js

@@ -43,7 +43,7 @@ class FileSharding {
   /**
    * @param ext the extension name of the output file (must starts with ".")
    */
-  createVirtualShardingFiles(ext = "") {
+  createVirtualShardingFiles(ext = "", index = 0) {
     const bf = Buffer.from(this.content, this.encoding);
     const res = [];
     const threshold = this.limitBytes;
@@ -52,16 +52,16 @@ class FileSharding {
       res.push(bf.subarray(tmpBytes, tmpBytes + threshold));
       tmpBytes += threshold;
     }
-    return res.map((e, i) => ({ filename: `${i}${ext}`, content: e }));
+    return res.map((e, i) => ({ filename: `${i + index}${ext}`, content: e }));
   }
   /**
    * @param folder absolute path of folder which used to save string sharding files.
    * @param ext the extension name of the output file (must starts with ".")
    */
-  async writeStringToFolder(folder, ext = "") {
+  async writeStringToFolder(folder, ext = "", index) {
     const dist = import_path.default.resolve(folder);
     await import_fs_extra.default.ensureDir(dist);
-    const res = this.createVirtualShardingFiles(ext);
+    const res = this.createVirtualShardingFiles(ext, index);
     await Promise.all(
       res.map(
         (e) => new Promise((resolve, reject) => {

package/dist/cjs/build/json.js

@@ -23,28 +23,53 @@ __export(json_exports, {
 });
 module.exports = __toCommonJS(json_exports);
 var import_json_stream_stringify = require("json-stream-stringify");
-var import_stream = require("stream");
 var import_path = require("path");
 var import_common = require("../common");
+var import_stream = require("stream");
+const maxFileSize = 1024 * 1024 * 400;
 function stringify(json, replacer, space, cycle) {
+  const jsonList = [];
   if (json && typeof json === "object") {
     return new Promise((resolve, reject) => {
-      let res = "";
-      const pt = new import_stream.PassThrough();
       const stream = new import_json_stream_stringify.JsonStreamStringify(json, replacer, space, cycle);
-      pt.on("data", (chunk) => {
-        res += chunk;
-      });
-      pt.on("end", () => {
-        return resolve(res);
-      });
-      pt.on("error", (err) => {
-        return reject(err);
+      let currentLength = 0;
+      let currentContent = "";
+      const batchProcessor = new import_stream.Transform({
+        readableObjectMode: true,
+        transform(chunk, _encoding, callback) {
+          const lines = chunk.toString().split("\\n");
+          lines.forEach((line) => {
+            if (currentLength + line.length > maxFileSize) {
+              jsonList.push(currentContent);
+              currentContent = "";
+              currentLength = 0;
+            }
+            if (line.length) {
+              currentContent += line;
+              currentLength += line.length;
+            }
+          });
+          callback();
+        }
       });
-      stream.on("error", (err) => {
+      stream.pipe(batchProcessor).on("data", (line) => {
+        if (currentLength + line.length > maxFileSize) {
+          jsonList.push(currentContent);
+          currentContent = "";
+          currentLength = 0;
+        }
+        if (line.length) {
+          currentContent += line;
+          currentLength += line.length;
+        }
+      }).on("end", () => {
+        if (jsonList.length < 1) {
+          jsonList.push(currentContent);
+        }
+        resolve(jsonList);
+      }).on("error", (err) => {
         return reject(err);
       });
-      stream.pipe(pt);
     });
   }
   return Promise.resolve(JSON.stringify(json, replacer, space));

package/dist/esm/build/file/sharding.js

@@ -10,7 +10,7 @@ class FileSharding {
   /**
    * @param ext the extension name of the output file (must starts with ".")
    */
-  createVirtualShardingFiles(ext = "") {
+  createVirtualShardingFiles(ext = "", index = 0) {
     const bf = Buffer.from(this.content, this.encoding);
     const res = [];
     const threshold = this.limitBytes;
@@ -19,16 +19,16 @@ class FileSharding {
       res.push(bf.subarray(tmpBytes, tmpBytes + threshold));
       tmpBytes += threshold;
     }
-    return res.map((e, i) => ({ filename: `${i}${ext}`, content: e }));
+    return res.map((e, i) => ({ filename: `${i + index}${ext}`, content: e }));
   }
   /**
    * @param folder absolute path of folder which used to save string sharding files.
    * @param ext the extension name of the output file (must starts with ".")
    */
-  async writeStringToFolder(folder, ext = "") {
+  async writeStringToFolder(folder, ext = "", index) {
     const dist = path.resolve(folder);
     await fse.ensureDir(dist);
-    const res = this.createVirtualShardingFiles(ext);
+    const res = this.createVirtualShardingFiles(ext, index);
     await Promise.all(
       res.map(
         (e) => new Promise((resolve, reject) => {

package/dist/esm/build/json.js

@@ -1,26 +1,51 @@
 import { JsonStreamStringify } from "json-stream-stringify";
-import { PassThrough } from "stream";
 import { dirname, join } from "path";
 import { Package } from "../common";
+import { Transform } from "stream";
+const maxFileSize = 1024 * 1024 * 400;
 function stringify(json, replacer, space, cycle) {
+  const jsonList = [];
   if (json && typeof json === "object") {
     return new Promise((resolve, reject) => {
-      let res = "";
-      const pt = new PassThrough();
       const stream = new JsonStreamStringify(json, replacer, space, cycle);
-      pt.on("data", (chunk) => {
-        res += chunk;
+      let currentLength = 0;
+      let currentContent = "";
+      const batchProcessor = new Transform({
+        readableObjectMode: true,
+        transform(chunk, _encoding, callback) {
+          const lines = chunk.toString().split("\\n");
+          lines.forEach((line) => {
+            if (currentLength + line.length > maxFileSize) {
+              jsonList.push(currentContent);
+              currentContent = "";
+              currentLength = 0;
+            }
+            if (line.length) {
+              currentContent += line;
+              currentLength += line.length;
+            }
+          });
+          callback();
+        }
       });
-      pt.on("end", () => {
-        return resolve(res);
-      });
-      pt.on("error", (err) => {
-        return reject(err);
-      });
-      stream.on("error", (err) => {
+      stream.pipe(batchProcessor).on("data", (line) => {
+        if (currentLength + line.length > maxFileSize) {
+          jsonList.push(currentContent);
+          currentContent = "";
+          currentLength = 0;
+        }
+        if (line.length) {
+          currentContent += line;
+          currentLength += line.length;
+        }
+      }).on("end", () => {
+        if (jsonList.length < 1) {
+          jsonList.push(currentContent);
+        }
+        resolve(jsonList);
+      }).on("error", (err) => {
         return reject(err);
       });
-      stream.pipe(pt);
     });
   }
   return Promise.resolve(JSON.stringify(json, replacer, space));

package/dist/type/build/file/sharding.d.ts

@@ -7,7 +7,7 @@ export declare class FileSharding {
     /**
      * @param ext the extension name of the output file (must starts with ".")
      */
-    createVirtualShardingFiles(ext?: string): {
+    createVirtualShardingFiles(ext?: string, index?: number): {
         filename: string;
         content: Buffer;
     }[];
@@ -15,7 +15,7 @@ export declare class FileSharding {
      * @param folder absolute path of folder which used to save string sharding files.
      * @param ext the extension name of the output file (must starts with ".")
      */
-    writeStringToFolder(folder: string, ext?: string): Promise<{
+    writeStringToFolder(folder: string, ext?: string, index?: number): Promise<{
         filename: string;
         content: Buffer;
     }[]>;

package/dist/type/build/file/sharding.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sharding.d.ts","sourceRoot":"","sources":["../../../../src/build/file/sharding.ts"],"names":[],"mappings":";AAIA,qBAAa,YAAY;IAErB,SAAS,CAAC,OAAO,EAAE,MAAM;IACzB,SAAS,CAAC,UAAU;IACpB,SAAS,CAAC,QAAQ,EAAE,cAAc;gBAFxB,OAAO,EAAE,MAAM,EACf,UAAU,SAAmB,EAC7B,QAAQ,GAAE,cAAwB;IAG9C;;OAEG;IACI,0BAA0B,CAAC,GAAG,SAAK;;;;IAc1C;;;OAGG;IACU,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAK;;;;CAsB1D"}
+{"version":3,"file":"sharding.d.ts","sourceRoot":"","sources":["../../../../src/build/file/sharding.ts"],"names":[],"mappings":";AAIA,qBAAa,YAAY;IAErB,SAAS,CAAC,OAAO,EAAE,MAAM;IACzB,SAAS,CAAC,UAAU;IACpB,SAAS,CAAC,QAAQ,EAAE,cAAc;gBAFxB,OAAO,EAAE,MAAM,EACf,UAAU,SAAmB,EAC7B,QAAQ,GAAE,cAAwB;IAG9C;;OAEG;IACI,0BAA0B,CAAC,GAAG,SAAK,EAAE,KAAK,SAAI;;;;IAcrD;;;OAGG;IACU,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAK,EAAE,KAAK,CAAC,EAAE,MAAM;;;;CAsB1E"}

package/dist/type/build/json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/build/json.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAItC,wBAAgB,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,GAAG,SAAS,GAAG,MAAM,EACvE,IAAI,EAAE,CAAC,EACP,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,GAAG,EACtD,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EACvB,KAAK,CAAC,EAAE,OAAO,GACd,OAAO,CAAC,CAAC,CAAC,CA4BZ;AAGD,eAAO,MAAM,eAAe,SACpB,MAAM,aACD,IAAI,cAAc,KAC5B,IAAI,gBAAgB,GAAG,SAiCzB,CAAC"}
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/build/json.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAOtC,wBAAgB,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,SAAS,SAAS,GAAG,SAAS,GAAG,MAAM,EACvE,IAAI,EAAE,CAAC,EACP,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,GAAG,EACtD,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EACvB,KAAK,CAAC,EAAE,OAAO,GACd,OAAO,CAAC,CAAC,CAAC,CA6DZ;AAGD,eAAO,MAAM,eAAe,SACpB,MAAM,aACD,IAAI,cAAc,KAC5B,IAAI,gBAAgB,GAAG,SAiCzB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rsdoctor/utils",
-  "version": "0.3.6",
+  "version": "0.3.8-beta.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/web-infra-dev/rsdoctor",
@@ -84,7 +84,7 @@
     "lodash": "^4.17.21",
     "rslog": "^1.2.0",
     "strip-ansi": "^6.0.1",
-    "@rsdoctor/types": "0.3.6"
+    "@rsdoctor/types": "0.3.8-beta.0"
   },
   "devDependencies": {
     "@types/babel__code-frame": "7.0.3",