@rsdoctor/sdk 1.6.1 → 1.6.2

package/dist/index.cjs

@@ -6568,6 +6568,41 @@ var __webpack_modules__ = {
             return firstPkgDir || null;
         }
         __webpack_require__.n(dayjs_index_js_namespaceObject)()().format('YYYYMMDD');
+        var promises_ = __webpack_require__("node:fs/promises"), promises_default = __webpack_require__.n(promises_);
+        let external_node_stream_namespaceObject = require("node:stream");
+        var external_node_zlib_ = __webpack_require__("node:zlib");
+        async function writeManifestShards(jsonFragments, folder, options = {}) {
+            if (0 === jsonFragments.length) throw Error('Cannot write empty JSON fragments.');
+            let { index = 0, shardSize = 10485760 } = options;
+            if (!Number.isSafeInteger(shardSize) || shardSize <= 0) throw Error('Manifest shard size must be a positive integer.');
+            await promises_default().mkdir(folder, {
+                recursive: !0
+            });
+            let files = [], shardParts = [], shardLength = 0, flushShard = async ()=>{
+                if (0 === shardLength) return;
+                let basename = String(index + files.length), filePath = external_node_path_default().resolve(folder, basename);
+                await promises_default().writeFile(filePath, shardParts.join(''), 'utf8'), files.push({
+                    path: filePath,
+                    basename
+                }), shardParts = [], shardLength = 0;
+            }, appendEncodedText = async (encodedText)=>{
+                let offset = 0;
+                for(; offset < encodedText.length;){
+                    let length = Math.min(shardSize - shardLength, encodedText.length - offset);
+                    shardParts.push(encodedText.slice(offset, offset + length)), shardLength += length, offset += length, shardLength === shardSize && await flushShard();
+                }
+            }, compressor = external_node_stream_namespaceObject.Readable.from(jsonFragments, {
+                objectMode: !1
+            }).pipe((0, external_node_zlib_.createDeflate)()), base64Remainder = Buffer.alloc(0);
+            for await (let chunk of compressor){
+                let compressedChunk = Buffer.from(chunk), bytes = base64Remainder.length ? Buffer.concat([
+                    base64Remainder,
+                    compressedChunk
+                ]) : compressedChunk, encodableLength = bytes.length - bytes.length % 3;
+                encodableLength > 0 && await appendEncodedText(bytes.subarray(0, encodableLength).toString('base64')), base64Remainder = Buffer.from(bytes.subarray(encodableLength));
+            }
+            return base64Remainder.length > 0 && await appendEncodedText(base64Remainder.toString('base64')), await flushShard(), files;
+        }
         class SDKCore {
             constructor({ name, root }){
                 this.hooks = {
@@ -6654,20 +6689,21 @@ var __webpack_modules__ = {
                             return build_namespaceObject.Json.stringify(data);
                         }
                     })();
-                    if (Array.isArray(jsonStr)) {
-                        let fileOffset = 0;
-                        for (let str of jsonStr){
-                            let result = await this.writeToFolder(str, outputDir, key, fileOffset);
-                            fileOffset += result.files.length, urlsPromiseList.push(result);
-                        }
-                    } else urlsPromiseList.push(this.writeToFolder(jsonStr, outputDir, key));
+                    Array.isArray(jsonStr) ? urlsPromiseList.push(this.writeJsonFragmentsToFolder(jsonStr, outputDir, key)) : urlsPromiseList.push(this.writeToFolder(jsonStr, outputDir, key));
                 }
                 let dataUrls = await Promise.all(urlsPromiseList);
                 logger_namespaceObject.logger.debug(`SDKCore.writePieces extraConfig: ${JSON.stringify(this.extraConfig)}`, '[SDKCore.writePieces]'), this.cloudData = {
                     client: {
                         enableRoutes: this.getClientRoutes()
                     },
-                    data: dataUrls.reduce((t, item)=>(t[item.name] = Array.isArray(item.files) ? item.files.map((e)=>e.path).concat(t[item.name] || []) : item.files, t), {})
+                    data: dataUrls.reduce((t, item)=>{
+                        if (!Array.isArray(item.files)) return t[item.name] = item.files, t;
+                        let previous = t[item.name];
+                        return t[item.name] = [
+                            ...Array.isArray(previous) ? previous : [],
+                            ...item.files.map((e)=>e.path)
+                        ], t;
+                    }, {})
                 };
             }
             async writeManifest() {
@@ -6685,15 +6721,19 @@ var __webpack_modules__ = {
                 }), manifestDiskPath;
             }
             writeToFolder(jsonStr, dir, key, index) {
-                let sharding = new build_namespaceObject.File.FileSharding(common_namespaceObject.Algorithm.compressText(jsonStr)), folder = external_node_path_default().resolve(dir, key);
-                return sharding.writeStringToFolder(folder, '', index).then((item)=>({
-                        name: key,
-                        files: item.map((el)=>({
-                                path: external_node_path_default().resolve(folder, el.filename),
-                                basename: el.filename,
-                                content: el.content
-                            }))
-                    }));
+                return this.writeJsonFragmentsToFolder([
+                    jsonStr
+                ], dir, key, index);
+            }
+            async writeJsonFragmentsToFolder(jsonFragments, dir, key, index) {
+                if (0 === jsonFragments.length) throw Error(`Cannot write empty JSON fragments for "${key}".`);
+                let folder = external_node_path_default().resolve(dir, key);
+                return {
+                    name: key,
+                    files: await writeManifestShards(jsonFragments, folder, {
+                        index
+                    })
+                };
             }
         }
         let sdk_require = (0, external_module_namespaceObject.createRequire)(__rslib_import_meta_url__);

package/dist/index.js

@@ -27,6 +27,9 @@ import { randomBytes } from "crypto";
 import node_process from "node:process";
 import { AsyncSeriesHook } from "tapable";
 import dayjs from "../compiled/dayjs/index.js";
+import node_fs_promises from "node:fs/promises";
+import { Readable } from "node:stream";
+import { createDeflate } from "node:zlib";
 import { __webpack_require__ } from "./612.js";
 import { fileURLToPath as __rspack_fileURLToPath } from "node:url";
 let __rspack_createRequire_require = __rspack_createRequire(import.meta.url);
@@ -55,7 +58,7 @@ __webpack_require__.add({
         });
     },
     "../../node_modules/.pnpm/body-parser@2.2.2/node_modules/body-parser/lib/read.js" (module, __unused_rspack_exports, __webpack_require__) {
-        var createError = __webpack_require__("../../node_modules/.pnpm/http-errors@2.0.1/node_modules/http-errors/index.js"), getBody = __webpack_require__("../../node_modules/.pnpm/raw-body@3.0.2/node_modules/raw-body/index.js"), iconv = __webpack_require__("../../node_modules/.pnpm/iconv-lite@0.7.0/node_modules/iconv-lite/lib/index.js"), onFinished = __webpack_require__("../../node_modules/.pnpm/on-finished@2.4.1/node_modules/on-finished/index.js"), zlib = __webpack_require__("node:zlib"), hasBody = __webpack_require__("../../node_modules/.pnpm/type-is@2.0.1/node_modules/type-is/index.js").hasBody, { getCharset } = __webpack_require__("../../node_modules/.pnpm/body-parser@2.2.2/node_modules/body-parser/lib/utils.js");
+        var createError = __webpack_require__("../../node_modules/.pnpm/http-errors@2.0.1/node_modules/http-errors/index.js"), getBody = __webpack_require__("../../node_modules/.pnpm/raw-body@3.0.2/node_modules/raw-body/index.js"), iconv = __webpack_require__("../../node_modules/.pnpm/iconv-lite@0.7.0/node_modules/iconv-lite/lib/index.js"), onFinished = __webpack_require__("../../node_modules/.pnpm/on-finished@2.4.1/node_modules/on-finished/index.js"), zlib = __webpack_require__("node:zlib?1e70"), hasBody = __webpack_require__("../../node_modules/.pnpm/type-is@2.0.1/node_modules/type-is/index.js").hasBody, { getCharset } = __webpack_require__("../../node_modules/.pnpm/body-parser@2.2.2/node_modules/body-parser/lib/utils.js");
         module.exports = function(req, res, next, parse, debug, options) {
             if (onFinished.isFinished(req)) {
                 debug('body already parsed'), next();
@@ -5130,7 +5133,7 @@ __webpack_require__.add({
     "fs?8b28" (module) {
         module.exports = __rspack_createRequire_require("fs");
     },
-    "node:zlib" (module) {
+    "node:zlib?1e70" (module) {
         module.exports = __rspack_createRequire_require("node:zlib");
     },
     "os?da1a" (module) {
@@ -6560,6 +6563,38 @@ function findRoot(startDir = process.cwd()) {
     }
     return firstPkgDir || null;
 }
+async function writeManifestShards(jsonFragments, folder, options = {}) {
+    if (0 === jsonFragments.length) throw Error('Cannot write empty JSON fragments.');
+    let { index = 0, shardSize = 10485760 } = options;
+    if (!Number.isSafeInteger(shardSize) || shardSize <= 0) throw Error('Manifest shard size must be a positive integer.');
+    await node_fs_promises.mkdir(folder, {
+        recursive: !0
+    });
+    let files = [], shardParts = [], shardLength = 0, flushShard = async ()=>{
+        if (0 === shardLength) return;
+        let basename = String(index + files.length), filePath = node_path.resolve(folder, basename);
+        await node_fs_promises.writeFile(filePath, shardParts.join(''), 'utf8'), files.push({
+            path: filePath,
+            basename
+        }), shardParts = [], shardLength = 0;
+    }, appendEncodedText = async (encodedText)=>{
+        let offset = 0;
+        for(; offset < encodedText.length;){
+            let length = Math.min(shardSize - shardLength, encodedText.length - offset);
+            shardParts.push(encodedText.slice(offset, offset + length)), shardLength += length, offset += length, shardLength === shardSize && await flushShard();
+        }
+    }, compressor = Readable.from(jsonFragments, {
+        objectMode: !1
+    }).pipe(createDeflate()), base64Remainder = Buffer.alloc(0);
+    for await (let chunk of compressor){
+        let compressedChunk = Buffer.from(chunk), bytes = base64Remainder.length ? Buffer.concat([
+            base64Remainder,
+            compressedChunk
+        ]) : compressedChunk, encodableLength = bytes.length - bytes.length % 3;
+        encodableLength > 0 && await appendEncodedText(bytes.subarray(0, encodableLength).toString('base64')), base64Remainder = Buffer.from(bytes.subarray(encodableLength));
+    }
+    return base64Remainder.length > 0 && await appendEncodedText(base64Remainder.toString('base64')), await flushShard(), files;
+}
 dayjs().format('YYYYMMDD');
 class SDKCore {
     constructor({ name, root }){
@@ -6647,20 +6682,21 @@ class SDKCore {
                     return Json.stringify(data);
                 }
             })();
-            if (Array.isArray(jsonStr)) {
-                let fileOffset = 0;
-                for (let str of jsonStr){
-                    let result = await this.writeToFolder(str, outputDir, key, fileOffset);
-                    fileOffset += result.files.length, urlsPromiseList.push(result);
-                }
-            } else urlsPromiseList.push(this.writeToFolder(jsonStr, outputDir, key));
+            Array.isArray(jsonStr) ? urlsPromiseList.push(this.writeJsonFragmentsToFolder(jsonStr, outputDir, key)) : urlsPromiseList.push(this.writeToFolder(jsonStr, outputDir, key));
         }
         let dataUrls = await Promise.all(urlsPromiseList);
         logger.debug(`SDKCore.writePieces extraConfig: ${JSON.stringify(this.extraConfig)}`, '[SDKCore.writePieces]'), this.cloudData = {
             client: {
                 enableRoutes: this.getClientRoutes()
             },
-            data: dataUrls.reduce((t, item)=>(t[item.name] = Array.isArray(item.files) ? item.files.map((e)=>e.path).concat(t[item.name] || []) : item.files, t), {})
+            data: dataUrls.reduce((t, item)=>{
+                if (!Array.isArray(item.files)) return t[item.name] = item.files, t;
+                let previous = t[item.name];
+                return t[item.name] = [
+                    ...Array.isArray(previous) ? previous : [],
+                    ...item.files.map((e)=>e.path)
+                ], t;
+            }, {})
         };
     }
     async writeManifest() {
@@ -6678,15 +6714,19 @@ class SDKCore {
         }), manifestDiskPath;
     }
     writeToFolder(jsonStr, dir, key, index) {
-        let sharding = new File.FileSharding(Algorithm.compressText(jsonStr)), folder = node_path.resolve(dir, key);
-        return sharding.writeStringToFolder(folder, '', index).then((item)=>({
-                name: key,
-                files: item.map((el)=>({
-                        path: node_path.resolve(folder, el.filename),
-                        basename: el.filename,
-                        content: el.content
-                    }))
-            }));
+        return this.writeJsonFragmentsToFolder([
+            jsonStr
+        ], dir, key, index);
+    }
+    async writeJsonFragmentsToFolder(jsonFragments, dir, key, index) {
+        if (0 === jsonFragments.length) throw Error(`Cannot write empty JSON fragments for "${key}".`);
+        let folder = node_path.resolve(dir, key);
+        return {
+            name: key,
+            files: await writeManifestShards(jsonFragments, folder, {
+                index
+            })
+        };
     }
 }
 let source_map = __webpack_require__("../../node_modules/.pnpm/source-map@0.7.6/node_modules/source-map/source-map.js"), sdk_require = createRequire(import.meta.url);

package/dist/sdk/sdk/core.d.cts

@@ -29,5 +29,6 @@ export declare abstract class SDKCore<T extends RsdoctorSDKOptions> implements S
     protected writeManifest(): Promise<string>;
     saveManifest(data: Common.PlainObject, options: SDK.WriteStoreOptionsType): Promise<string>;
     protected writeToFolder(jsonStr: string, dir: string, key: string, index?: number): Promise<DataWithUrl>;
+    protected writeJsonFragmentsToFolder(jsonFragments: string[], dir: string, key: string, index?: number): Promise<DataWithUrl>;
     abstract onDataReport(): void | Promise<void>;
 }

package/dist/sdk/sdk/core.d.ts

@@ -29,5 +29,6 @@ export declare abstract class SDKCore<T extends RsdoctorSDKOptions> implements S
     protected writeManifest(): Promise<string>;
     saveManifest(data: Common.PlainObject, options: SDK.WriteStoreOptionsType): Promise<string>;
     protected writeToFolder(jsonStr: string, dir: string, key: string, index?: number): Promise<DataWithUrl>;
+    protected writeJsonFragmentsToFolder(jsonFragments: string[], dir: string, key: string, index?: number): Promise<DataWithUrl>;
     abstract onDataReport(): void | Promise<void>;
 }

package/dist/sdk/sdk/types.d.cts

@@ -4,7 +4,7 @@ export interface DataWithUrl {
     files: {
         path: string;
         basename: string;
-        content: Buffer;
+        content?: Buffer;
     }[] | string;
 }
 export interface RsdoctorSDKOptions {

package/dist/sdk/sdk/types.d.ts

@@ -4,7 +4,7 @@ export interface DataWithUrl {
     files: {
         path: string;
         basename: string;
-        content: Buffer;
+        content?: Buffer;
     }[] | string;
 }
 export interface RsdoctorSDKOptions {

package/dist/sdk/utils/index.d.cts

@@ -1,3 +1,4 @@
 export * from './constant';
 export * from './base';
 export * from './upload';
+export * from './manifestSharding';

package/dist/sdk/utils/index.d.ts

@@ -1,3 +1,4 @@
 export * from './constant.js';
 export * from './base.js';
 export * from './upload.js';
+export * from './manifestSharding.js';

package/dist/sdk/utils/manifestSharding.d.cts

@@ -0,0 +1,15 @@
+export declare const DEFAULT_MANIFEST_SHARD_SIZE: number;
+interface WriteManifestShardsOptions {
+    index?: number;
+    shardSize?: number;
+}
+export interface ManifestShardFile {
+    path: string;
+    basename: string;
+}
+/**
+ * Compress JSON fragments as one deflate stream, encode that stream as one
+ * continuous Base64 value, and write bounded physical shards.
+ */
+export declare function writeManifestShards(jsonFragments: string[], folder: string, options?: WriteManifestShardsOptions): Promise<ManifestShardFile[]>;
+export {};

package/dist/sdk/utils/manifestSharding.d.ts

@@ -0,0 +1,15 @@
+export declare const DEFAULT_MANIFEST_SHARD_SIZE: number;
+interface WriteManifestShardsOptions {
+    index?: number;
+    shardSize?: number;
+}
+export interface ManifestShardFile {
+    path: string;
+    basename: string;
+}
+/**
+ * Compress JSON fragments as one deflate stream, encode that stream as one
+ * continuous Base64 value, and write bounded physical shards.
+ */
+export declare function writeManifestShards(jsonFragments: string[], folder: string, options?: WriteManifestShardsOptions): Promise<ManifestShardFile[]>;
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rsdoctor/sdk",
-  "version": "1.6.1",
+  "version": "1.6.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/web-infra-dev/rsdoctor",
@@ -32,10 +32,10 @@
     "safer-buffer": "2.1.2",
     "socket.io": "4.8.1",
     "tapable": "2.3.3",
-    "@rsdoctor/client": "1.6.1",
-    "@rsdoctor/graph": "1.6.1",
-    "@rsdoctor/utils": "1.6.1",
-    "@rsdoctor/types": "1.6.1"
+    "@rsdoctor/client": "1.6.2",
+    "@rsdoctor/graph": "1.6.2",
+    "@rsdoctor/utils": "1.6.2",
+    "@rsdoctor/types": "1.6.2"
   },
   "devDependencies": {
     "@types/body-parser": "1.19.6",