@soga/part-uploader 0.0.1 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/main.d.ts +11 -0
- package/dist/main.js +1 -0
- package/dist/types/main.d.ts +17 -0
- package/dist/types/main.js +1 -0
- package/dist/uploader/ali-helper.d.ts +34 -0
- package/dist/uploader/ali-helper.js +1 -0
- package/dist/uploader/ali-uploader.d.ts +28 -0
- package/dist/uploader/ali-uploader.js +1 -0
- package/dist/uploader/baidu-helper.d.ts +16 -0
- package/dist/uploader/baidu-helper.js +1 -0
- package/dist/uploader/baidu-uploader.d.ts +23 -0
- package/dist/uploader/baidu-uploader.js +1 -0
- package/dist/uploader/base-uploader.d.ts +33 -0
- package/dist/uploader/base-uploader.js +1 -0
- package/dist/utils/chunk.d.ts +14 -0
- package/dist/utils/chunk.js +1 -0
- package/package.json +48 -4
- package/index.js +0 -1
package/README.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# single-uploader
|
package/dist/main.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { UploadAliParams, UploadBaiduParams } from './types/main';
|
|
2
|
+
export { BaiduUploader } from './uploader/baidu-uploader';
|
|
3
|
+
export { AliUploader } from './uploader/ali-uploader';
|
|
4
|
+
export declare const uploadAli: (params: UploadAliParams) => Promise<{
|
|
5
|
+
drive_id: string;
|
|
6
|
+
file_id: string;
|
|
7
|
+
}>;
|
|
8
|
+
export declare const uploadBaidu: (params: UploadBaiduParams) => Promise<{
|
|
9
|
+
fs_id: number;
|
|
10
|
+
}>;
|
|
11
|
+
export * from './types/main';
|
package/dist/main.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,r,a,o){void 0===o&&(o=a);var t=Object.getOwnPropertyDescriptor(r,a);t&&!("get"in t?!r.__esModule:t.writable||t.configurable)||(t={enumerable:!0,get:function(){return r[a]}}),Object.defineProperty(e,o,t)}:function(e,r,a,o){void 0===o&&(o=a),e[o]=r[a]}),__exportStar=this&&this.__exportStar||function(e,r){for(var a in e)"default"===a||Object.prototype.hasOwnProperty.call(r,a)||__createBinding(r,e,a)};Object.defineProperty(exports,"__esModule",{value:!0}),exports.uploadBaidu=exports.uploadAli=exports.AliUploader=exports.BaiduUploader=void 0;const ali_uploader_1=require("./uploader/ali-uploader"),baidu_uploader_1=require("./uploader/baidu-uploader");var baidu_uploader_2=require("./uploader/baidu-uploader");Object.defineProperty(exports,"BaiduUploader",{enumerable:!0,get:function(){return baidu_uploader_2.BaiduUploader}});var ali_uploader_2=require("./uploader/ali-uploader");Object.defineProperty(exports,"AliUploader",{enumerable:!0,get:function(){return ali_uploader_2.AliUploader}});const uploadAli=async e=>{const r=new ali_uploader_1.AliUploader(e);return await r.start()};exports.uploadAli=uploadAli;const uploadBaidu=async e=>{const r=new baidu_uploader_1.BaiduUploader(e);return await r.start()};exports.uploadBaidu=uploadBaidu,__exportStar(require("./types/main"),exports);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { FilePartItem } from '@soga/types';
|
|
2
|
+
export type CommonPartUploaderParams = {
|
|
3
|
+
file_id: number;
|
|
4
|
+
host_id: number;
|
|
5
|
+
part_id: number;
|
|
6
|
+
output_root: string;
|
|
7
|
+
part: FilePartItem;
|
|
8
|
+
sdk_domain: string;
|
|
9
|
+
};
|
|
10
|
+
export type UploadBaiduParams = CommonPartUploaderParams & {
|
|
11
|
+
cloud_folder_path: string;
|
|
12
|
+
ua?: string;
|
|
13
|
+
};
|
|
14
|
+
export type UploadAliParams = CommonPartUploaderParams & {
|
|
15
|
+
cloud_folder_id: string;
|
|
16
|
+
ua?: string;
|
|
17
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { AxiosInstance } from 'axios';
|
|
2
|
+
import { UploadAliParams } from '../types/main';
|
|
3
|
+
import { CompleteGroupList } from '../utils/chunk';
|
|
4
|
+
import { BaseUploader } from './base-uploader';
|
|
5
|
+
export declare class AliHelper extends BaseUploader {
|
|
6
|
+
params: UploadAliParams;
|
|
7
|
+
protected drive_id: string;
|
|
8
|
+
protected axios: AxiosInstance;
|
|
9
|
+
protected domain: string;
|
|
10
|
+
protected chunk_size: number;
|
|
11
|
+
protected group_list: CompleteGroupList;
|
|
12
|
+
constructor(params: UploadAliParams);
|
|
13
|
+
protected getExpiredAt(): number;
|
|
14
|
+
private initAxios;
|
|
15
|
+
protected init(): Promise<void>;
|
|
16
|
+
protected getUploadId(): Promise<string>;
|
|
17
|
+
protected cacheUploadId(upload_id: string): Promise<void>;
|
|
18
|
+
protected clearCache(): Promise<void>;
|
|
19
|
+
protected getFileId(): Promise<string>;
|
|
20
|
+
protected cacheFileId(file_id: string): Promise<void>;
|
|
21
|
+
protected getPartInfoList(): Promise<{
|
|
22
|
+
state: boolean;
|
|
23
|
+
expired_at: number;
|
|
24
|
+
data: {
|
|
25
|
+
part_number: number;
|
|
26
|
+
upload_url: string;
|
|
27
|
+
}[];
|
|
28
|
+
}>;
|
|
29
|
+
protected cachePartInfoList(part_info_list: {
|
|
30
|
+
part_number: number;
|
|
31
|
+
upload_url: string;
|
|
32
|
+
}[]): Promise<void>;
|
|
33
|
+
protected getProofCode(): Promise<string>;
|
|
34
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.AliHelper=void 0;const axios_1=__importDefault(require("axios")),base_uploader_1=require("./base-uploader"),crypto_1=require("crypto"),utils_1=require("@soga/utils"),types_1=require("@soga/types"),UPLOAD_ID="upload-id",FILE_ID="file-id",PART_INFO_LIST="part-info-list";class AliHelper extends base_uploader_1.BaseUploader{params;drive_id;axios;domain="https://openapi.alipan.com";chunk_size=4194304;group_list=[];constructor(t){super(t,types_1.HostType.ALI),this.params=t,this.initAxios()}getExpiredAt(){return Date.now()+348e4}initAxios(){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.request.use((async t=>{const{access_token:e}=await this.sdk.getAliHostAuthData({host_id:this.params.host_id,refresh:!1});return t.headers.Authorization=`Bearer ${e}`,t}),(t=>Promise.reject(t)))}async init(){await super.init();const{drive_id:t}=await this.sdk.getAliHostAuthData({host_id:this.params.host_id,refresh:!1});this.drive_id=t}async getUploadId(){const t=UPLOAD_ID,e=await this.db.get(t);if(e)return e}async cacheUploadId(t){const e=UPLOAD_ID;await this.db.put(e,t)}async clearCache(){await this.db.del(UPLOAD_ID),await this.db.del(FILE_ID),await this.db.del(this.GROUP_LIST_KEY),await this.db.del(PART_INFO_LIST)}async getFileId(){const t=FILE_ID,e=await this.db.get(t);if(e)return e}async cacheFileId(t){const e=FILE_ID;await this.db.put(e,t)}async getPartInfoList(){const t=PART_INFO_LIST,e=await this.db.get(t);if(e){const{expired_at:t,part_info_list:i}=e;return Date.now()<t?{state:!0,expired_at:t,data:i}:{state:!1,expired_at:t,data:i}}}async cachePartInfoList(t){const e=PART_INFO_LIST;await this.db.put(e,{expired_at:this.getExpiredAt(),part_info_list:t})}async getProofCode(){const{access_token:t}=await this.sdk.getAliHostAuthData({host_id:this.params.host_id,refresh:!1}),{part:e}=this.params,i=e.size,s=(0,crypto_1.createHash)("md5").update(t).digest("hex").slice(0,16),a=BigInt(`0x${s}`),r=Number(a%BigInt(i)),o=r,d=Math.min(i,r+8),_=e.start+o,n=e.start+d;return(await(0,utils_1.getFileBufferSlice)(e.path,_,n-1)).toString("base64")}}exports.AliHelper=AliHelper;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { AliHelper } from './ali-helper';
|
|
2
|
+
export declare class AliUploader extends AliHelper {
|
|
3
|
+
private queue_expired_at;
|
|
4
|
+
private queue_list;
|
|
5
|
+
start(): Promise<{
|
|
6
|
+
drive_id: string;
|
|
7
|
+
file_id: string;
|
|
8
|
+
}>;
|
|
9
|
+
protected postcreate(): Promise<{
|
|
10
|
+
file_id: string;
|
|
11
|
+
}>;
|
|
12
|
+
protected uploadChunks(): Promise<void>;
|
|
13
|
+
protected uploadOneChunk({ upload_url, part_number, }: {
|
|
14
|
+
upload_url: string;
|
|
15
|
+
part_number: number;
|
|
16
|
+
}): Promise<void>;
|
|
17
|
+
protected precreate(): Promise<{
|
|
18
|
+
need_upload: boolean;
|
|
19
|
+
data: {
|
|
20
|
+
file_id: string;
|
|
21
|
+
};
|
|
22
|
+
} | {
|
|
23
|
+
need_upload: boolean;
|
|
24
|
+
data?: undefined;
|
|
25
|
+
}>;
|
|
26
|
+
protected correctUploaded(upload_id: string): Promise<void>;
|
|
27
|
+
protected autoRefreshUploadUrl(): Promise<void>;
|
|
28
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.AliUploader=void 0;const fs_1=require("fs"),ali_helper_1=require("./ali-helper"),axios_1=__importDefault(require("axios")),types_1=require("@soga/types");class AliUploader extends ali_helper_1.AliHelper{queue_expired_at;queue_list;async start(){try{await this.init();const t=await this.getUploadId();if(t)await this.correctUploaded(t);else{const{need_upload:t,data:e}=await this.precreate();if(!t)return await this.clearCache(),await this.postProgress({type:types_1.UploadProcessStep.upload_ali,percent:1,host_type:types_1.HostType.ALI}),{...e,drive_id:this.drive_id}}await this.uploadChunks();const e=await this.postcreate();return await this.clearCache(),await this.postProgress({type:types_1.UploadProcessStep.upload_ali,percent:1,host_type:types_1.HostType.ALI}),{...e,drive_id:this.drive_id}}catch(t){if("ManualStopError"!=t.name&&"CanceledError"!=t.name)throw t}finally{this.db&&await this.db.close()}}async postcreate(){const t=await this.getFileId(),e=await this.getUploadId(),a=await this.axios.post("/adrive/v1.0/openFile/complete",{drive_id:this.drive_id,file_id:t,upload_id:e},{signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:16300,message:t.message,stack:t.stack})}));if(a)return{file_id:a.data.file_id}}async uploadChunks(){this.queue_list;const t=async()=>{if(this.queue_list.length){await this.autoRefreshUploadUrl();const e=this.queue_list.shift();await this.uploadOneChunk(e);const a=Math.min(e.part_number*this.chunk_size,this.params.part.size)/this.params.part.size;await this.postProgress({type:types_1.UploadProcessStep.upload_ali,percent:.97*a,host_type:types_1.HostType.ALI}),await t()}};await t()}async uploadOneChunk({upload_url:t,part_number:e}){const a=e-1,i=this.group_list[a],s=this.params.part.path,r=(0,fs_1.createReadStream)(s,{start:i.start,end:i.end});await axios_1.default.put(t,r,{headers:{"Content-Type":null,"User-Agent":this.params.ua},timeout:12e4,signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:16200,message:t.message,stack:t.stack})}))}async precreate(){const t=await this.getProofCode(),e=this.group_list.map(((t,e)=>({part_number:e+1}))),{part:a}=this.params,i=`dpan_p_${a.md5}.txt`,s={drive_id:this.drive_id,parent_file_id:this.params.cloud_folder_id,name:i,type:"file",check_name_mode:"refuse",part_info_list:e,size:a.size,content_hash_name:"sha1",content_hash:a.sha1,proof_code:t,proof_version:"v1"},r=await this.axios.post("/adrive/v1.0/openFile/create",s,{signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:16100,message:t.message,stack:t.stack})}));if(!r)return;const{data:o}=r;return o.exist&&"available"===o.status||o.rapid_upload?{need_upload:!1,data:{file_id:o.file_id}}:(await this.cacheFileId(o.file_id),await this.cacheUploadId(o.upload_id),await this.cachePartInfoList(o.part_info_list),this.queue_list=o.part_info_list,this.queue_expired_at=this.getExpiredAt(),{need_upload:!0})}async correctUploaded(t){const e=await this.getFileId(),a=await this.axios.post("/adrive/v1.0/openFile/listUploadedParts",{drive_id:this.drive_id,upload_id:t,file_id:e},{signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:16110,message:t.message,stack:t.stack})}));if(!a)return;const i=a.data.uploaded_parts.map((t=>t.part_number)),s=await this.getPartInfoList();if(s){const t=s.data.filter((t=>!i.includes(t.part_number)));this.queue_expired_at=s.expired_at,this.queue_list=t,await this.autoRefreshUploadUrl()}}async autoRefreshUploadUrl(){const{queue_expired_at:t,queue_list:e}=this;if(t>Date.now())return;const a=e.map((t=>({part_number:t.part_number}))),i={drive_id:this.drive_id,upload_id:await this.getUploadId(),file_id:await this.getFileId(),part_info_list:a},s=await this.axios.post("/adrive/v1.0/openFile/getUploadUrl",i,{signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:16120,message:t.message,stack:t.stack})}));s&&(this.queue_expired_at=this.getExpiredAt(),this.queue_list=s.data.part_info_list,await this.cachePartInfoList(s.data.part_info_list))}}exports.AliUploader=AliUploader;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { UploadBaiduParams } from '../types/main';
|
|
2
|
+
import { BaseUploader } from './base-uploader';
|
|
3
|
+
import { CompleteGroupList } from '../utils/chunk';
|
|
4
|
+
export declare class BaiduHelper extends BaseUploader {
|
|
5
|
+
protected chunk_size: number;
|
|
6
|
+
protected group_list: CompleteGroupList;
|
|
7
|
+
protected md5_list: string[];
|
|
8
|
+
protected params: UploadBaiduParams;
|
|
9
|
+
constructor(params: UploadBaiduParams);
|
|
10
|
+
protected init(): Promise<void>;
|
|
11
|
+
protected initMd5List(): Promise<void>;
|
|
12
|
+
protected getSearchParams(payload: Record<string, string>): string;
|
|
13
|
+
protected getUploadId(): Promise<string>;
|
|
14
|
+
protected cacheUploadId(upload_id: string): Promise<void>;
|
|
15
|
+
protected clearCache(): Promise<void>;
|
|
16
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaiduHelper=void 0;const utils_1=require("@soga/utils"),base_uploader_1=require("./base-uploader"),types_1=require("@soga/types"),UPLOAD_ID="upload-id",MD5_LIST="md5-list";class BaiduHelper extends base_uploader_1.BaseUploader{chunk_size=4194304;group_list=[];md5_list=[];params;constructor(t){super(t,types_1.HostType.BAIDU),this.params=t}async init(){await super.init(),await this.initMd5List()}async initMd5List(){const t=MD5_LIST,s=await this.db.get(t);if(s)return void(this.md5_list=s);const e=this.group_list,a=[];for(const t of e){const{path:s}=this.params.part,e=await(0,utils_1.calculateMd5)({file:s,start:t.start,end:t.end});a.push(e)}await this.db.put(t,a),this.md5_list=a}getSearchParams(t){Object.keys(t).forEach((s=>{void 0===t[s]&&delete t[s]}));return new URLSearchParams(t).toString()}async getUploadId(){const t=UPLOAD_ID,s=await this.db.get(t);if(s)return s}async cacheUploadId(t){const s=UPLOAD_ID;await this.db.put(s,t)}async clearCache(){await this.db.del(UPLOAD_ID),await this.db.del(this.GROUP_LIST_KEY),await this.db.del(MD5_LIST)}}exports.BaiduHelper=BaiduHelper;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { BaiduHelper } from './baidu-helper';
|
|
2
|
+
export declare class BaiduUploader extends BaiduHelper {
|
|
3
|
+
private queue_list;
|
|
4
|
+
protected get cloudPath(): string;
|
|
5
|
+
start(): Promise<{
|
|
6
|
+
fs_id: number;
|
|
7
|
+
}>;
|
|
8
|
+
protected postcreate(): Promise<{
|
|
9
|
+
fs_id: number;
|
|
10
|
+
}>;
|
|
11
|
+
protected uploadChunks(): Promise<void>;
|
|
12
|
+
protected precreate(): Promise<{
|
|
13
|
+
need_upload: boolean;
|
|
14
|
+
data?: undefined;
|
|
15
|
+
} | {
|
|
16
|
+
need_upload: boolean;
|
|
17
|
+
data: {
|
|
18
|
+
fs_id: number;
|
|
19
|
+
};
|
|
20
|
+
}>;
|
|
21
|
+
private uploadOneChunk;
|
|
22
|
+
private getChunkInfo;
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaiduUploader=void 0;const form_data_1=__importDefault(require("form-data")),utils_1=require("@soga/utils"),baidu_helper_1=require("./baidu-helper"),axios_1=__importDefault(require("axios")),path_1=require("path"),types_1=require("@soga/types");class BaiduUploader extends baidu_helper_1.BaiduHelper{queue_list=[];get cloudPath(){return`${this.params.cloud_folder_path}/dpan_p_${this.params.part.md5}.bin`}async start(){try{await this.init();const t=await this.precreate();let a=t.data;return t.need_upload&&(await this.uploadChunks(),a=await this.postcreate()),await this.clearCache(),await this.postProgress({type:types_1.UploadProcessStep.upload_baidu,percent:1,host_type:types_1.HostType.BAIDU}),a}finally{this.db&&await this.db.close()}}async postcreate(){const t=await this.getUploadId(),a=this.cloudPath,e=this.md5_list,s=this.params.part.size,{access_token:i}=await this.sdk.getBaiduHostAuthData({host_id:this.params.host_id,refresh:!1}),r=`https://pan.baidu.com/rest/2.0/xpan/file?method=create&access_token=${i}`,o=this.getSearchParams({path:a,size:`${s}`,isdir:"0",block_list:JSON.stringify(e),uploadid:t||void 0,rtype:"2"}),d=await axios_1.default.post(r,o,{signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:15300,message:t.message,stack:t.stack})}));if(!d)return;const{data:p}=d;if(!p.errno)return{fs_id:p.fs_id};{const t=`upload create file error: ${p.errno}`;await this.throwError({code:15300,message:t,stack:JSON.stringify(p)})}}async uploadChunks(){const t=this.queue_list.map((t=>t)),a=async()=>{if(t.length){const e=t.shift(),s=await this.getChunkInfo(e);await this.uploadOneChunk(s);const i=Math.min((e+1)*this.chunk_size,this.params.part.size)/this.params.part.size;await this.postProgress({type:types_1.UploadProcessStep.upload_baidu,percent:.98*i,host_type:types_1.HostType.BAIDU}),await a()}};await a()}async precreate(){const t=this.md5_list,{md4:a,md5:e,size:s}=this.params.part,{access_token:i}=await this.sdk.getBaiduHostAuthData({host_id:this.params.host_id,refresh:!1}),r=await this.getUploadId(),o=`https://pan.baidu.com/rest/2.0/xpan/file?method=precreate&access_token=${i}`,d=this.getSearchParams({path:this.cloudPath,size:`${s}`,isdir:"0",block_list:JSON.stringify(t),autoinit:"1",rtype:"2","content-md5":e,"slice-md5":a,uploadid:r}),p=await axios_1.default.post(o,d,{headers:{"User-Agent":this.params.ua},signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:15100,message:t.message,stack:t.stack})}));if(!p)return;const{data:h}=p;if(!h.errno)return 1===h.return_type?(h.uploadid&&h.uploadid!==r&&await this.cacheUploadId(h.uploadid),this.queue_list=h.block_list,{need_upload:!0}):{need_upload:!1,data:{fs_id:h.info.fs_id}};{const t=`precreate error: ${h.errno}`;await this.throwError({code:15100,message:t,stack:JSON.stringify(h)})}}async uploadOneChunk(t){const a=await this.getUploadId(),e=this.cloudPath,{access_token:s}=await this.sdk.getBaiduHostAuthData({host_id:this.params.host_id,refresh:!1}),i=`https://d.pcs.baidu.com/rest/2.0/pcs/superfile2?access_token=${s}&method=upload&type=tmpfile&path=${e}&uploadid=${a}&partseq=${t.index}`,r=new form_data_1.default;r.append("file",t.buffer,`${t.filename}`);const o=await axios_1.default.post(i,r,{timeout:3e5,signal:this.controller.signal}).catch((async t=>{if("CanceledError"==t.name)throw t;await this.throwError({code:15200,message:t.message,stack:t.stack})}));if(o)return o.data.md5}async getChunkInfo(t){const a=this.group_list[t],e=this.params.part.path,s=await(0,utils_1.getFileBufferSlice)(e,a.start,a.end),{md5_list:i}=this;return{index:t,buffer:s,md5:i[t],filename:(0,path_1.parse)(e).base,size:s.byteLength}}}exports.BaiduUploader=BaiduUploader;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Level } from 'level';
|
|
2
|
+
import { CommonPartUploaderParams } from '../types/main';
|
|
3
|
+
import { HostType, UploadProcessStep, UploadWorkerMessage } from '@soga/types';
|
|
4
|
+
import { Sdk } from '@soga/sdk';
|
|
5
|
+
import { CompleteGroupList } from '../utils/chunk';
|
|
6
|
+
type LevelDataType = string | number | boolean | object | null;
|
|
7
|
+
export declare class BaseUploader {
|
|
8
|
+
protected GROUP_LIST_KEY: string;
|
|
9
|
+
protected chunk_size: number;
|
|
10
|
+
protected group_list: CompleteGroupList;
|
|
11
|
+
protected host_type: HostType;
|
|
12
|
+
protected controller: AbortController;
|
|
13
|
+
protected is_stop: boolean;
|
|
14
|
+
protected db: Level<string, LevelDataType>;
|
|
15
|
+
protected sdk: Sdk;
|
|
16
|
+
protected baseParams: CommonPartUploaderParams;
|
|
17
|
+
constructor(params: CommonPartUploaderParams, host_type: HostType);
|
|
18
|
+
protected init(): Promise<void>;
|
|
19
|
+
protected initGroupList(): Promise<void>;
|
|
20
|
+
private initLevel;
|
|
21
|
+
throwError(error: {
|
|
22
|
+
code: number;
|
|
23
|
+
message: string;
|
|
24
|
+
stack?: string;
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
postProgress(progress: {
|
|
27
|
+
type: UploadProcessStep;
|
|
28
|
+
percent: number;
|
|
29
|
+
host_type: HostType;
|
|
30
|
+
}): Promise<void>;
|
|
31
|
+
protected postMessage(data: UploadWorkerMessage): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const utils_1=require("@soga/utils"),level_1=require("level"),path_1=require("path"),sdk_1=require("@soga/sdk"),worker_threads_1=require("worker_threads"),chunk_1=require("../utils/chunk");class BaseUploader{GROUP_LIST_KEY="group_list";chunk_size=4194304;group_list=[];host_type;controller=new AbortController;is_stop=!1;db;sdk;baseParams;constructor(s,t){this.host_type=t,this.baseParams=s}async init(){this.initLevel(),this.sdk=(0,sdk_1.getSdk)(this.baseParams.sdk_domain),await this.initGroupList()}async initGroupList(){const s=await this.db.get(this.GROUP_LIST_KEY);if(s)return void(this.group_list=s);const t=(0,chunk_1.groupChunk)(this.baseParams.part,this.chunk_size);this.group_list=t,await this.db.put(this.GROUP_LIST_KEY,t)}initLevel(){const s=`upload_part_${this.host_type}_${this.baseParams.part_id}`,t=(0,path_1.resolve)(this.baseParams.output_root,s),e=new level_1.Level(t,{valueEncoding:"json"});this.db=e}async throwError(s){throw new utils_1.SimpleError(s)}async postProgress(s){await this.postMessage({id:this.baseParams.file_id,type:"percent",data:{part_id:this.baseParams.part_id,host_type:this.host_type,percent:s.percent}})}async postMessage(s){worker_threads_1.parentPort&&worker_threads_1.parentPort.postMessage(s)}}exports.BaseUploader=BaseUploader;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { FilePartItem } from '@soga/types';
|
|
2
|
+
export declare function groupChunk(fileItem: FilePartItem, chunkSize?: number): CompleteGroupList;
|
|
3
|
+
export type GroupItem = {
|
|
4
|
+
index: number;
|
|
5
|
+
file: string;
|
|
6
|
+
start: number;
|
|
7
|
+
end: number;
|
|
8
|
+
finish: boolean;
|
|
9
|
+
};
|
|
10
|
+
type CompleteGroupItem = GroupItem & {
|
|
11
|
+
size: number;
|
|
12
|
+
};
|
|
13
|
+
export type CompleteGroupList = CompleteGroupItem[];
|
|
14
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";function groupChunk(e,t=4194304){const s=[],i=e.size;if(i<=t)s.push({file:e.file,start:e.start,end:e.end,finish:!0});else{const n=i/t,r=Math.floor(n),f=n===r;for(let i=0;i<r;i++)s.push({file:e.file,start:e.start+i*t,end:e.start+(i+1)*t-1,finish:!!f&&i===r});if(!f){const i=e.start+r*t;s.push({file:e.file,start:i,end:e.end,finish:!0})}}return s.map(((e,t)=>({...e,size:e.end-e.start+1,index:t})))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.groupChunk=groupChunk;
|
package/package.json
CHANGED
|
@@ -1,15 +1,59 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soga/part-uploader",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"main": "index.js",
|
|
3
|
+
"version": "0.2.8",
|
|
5
4
|
"publishConfig": {
|
|
6
5
|
"access": "public"
|
|
7
6
|
},
|
|
7
|
+
"description": "",
|
|
8
|
+
"main": "dist/main.js",
|
|
9
|
+
"types": "dist/main.d.ts",
|
|
10
|
+
"files": [
|
|
11
|
+
"dist"
|
|
12
|
+
],
|
|
8
13
|
"scripts": {
|
|
9
|
-
"
|
|
14
|
+
"build": "rimraf dist && tsc && ts-node ./scripts/minify",
|
|
15
|
+
"minify": "ts-node ./scripts/minify",
|
|
16
|
+
"demo_backup": "ts-node ./demo/demo.ts",
|
|
17
|
+
"demo": "ts-node ./demo/demo.ts",
|
|
18
|
+
"worker": "tsc && ts-node ./demo/worker.ts",
|
|
19
|
+
"test": "jest",
|
|
20
|
+
"dev": "ts-node ./src/main.ts",
|
|
21
|
+
"lint": "eslint . --ext .ts",
|
|
22
|
+
"prepublishOnly": "npm run build"
|
|
23
|
+
},
|
|
24
|
+
"devDependencies": {
|
|
25
|
+
"@types/fs-extra": "^11.0.4",
|
|
26
|
+
"@types/glob": "^8.1.0",
|
|
27
|
+
"@types/jest": "^29.5.14",
|
|
28
|
+
"@types/node": "^20.8.7",
|
|
29
|
+
"@typescript-eslint/eslint-plugin": "^6.4.1",
|
|
30
|
+
"@typescript-eslint/parser": "^6.4.1",
|
|
31
|
+
"eslint": "^8.47.0",
|
|
32
|
+
"eslint-config-prettier": "^9.1.0",
|
|
33
|
+
"eslint-plugin-jest": "^27.9.0",
|
|
34
|
+
"eslint-plugin-prettier": "^5.2.3",
|
|
35
|
+
"glob": "^10.4.5",
|
|
36
|
+
"jest": "^29.7.0",
|
|
37
|
+
"prettier": "^3.0.2",
|
|
38
|
+
"rimraf": "^6.0.1",
|
|
39
|
+
"terser": "^5.19.2",
|
|
40
|
+
"ts-jest": "^29.2.5",
|
|
41
|
+
"ts-node": "^10.9.1",
|
|
42
|
+
"typescript": "^5.1.6"
|
|
10
43
|
},
|
|
11
44
|
"keywords": [],
|
|
12
45
|
"author": "",
|
|
13
46
|
"license": "ISC",
|
|
14
|
-
"
|
|
47
|
+
"dependencies": {
|
|
48
|
+
"@soga/sdk": "file:../base-sdk",
|
|
49
|
+
"@soga/types": "file:../base-types",
|
|
50
|
+
"@soga/utils": "file:../base-utils",
|
|
51
|
+
"axios": "^1.7.9",
|
|
52
|
+
"form-data": "^4.0.2",
|
|
53
|
+
"fs-extra": "^11.2.0"
|
|
54
|
+
},
|
|
55
|
+
"peerDependencies": {
|
|
56
|
+
"level": "*"
|
|
57
|
+
},
|
|
58
|
+
"gitHead": "2f239908854aa1f8ad224b83316d0c33c20feea1"
|
|
15
59
|
}
|
package/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
console.log("cache");
|