@soga/subtitles 0.3.0 → 1.0.0

package/dist/main.d.mts

@@ -0,0 +1,24 @@
+import { DataSource } from 'typeorm';
+
+interface CommonSubtitlesParams {
+    ffmpegPath: string;
+    sdk_domain: string;
+    dataSource: DataSource;
+    id: number;
+    debug?: boolean;
+}
+interface AddSubtitlesParams extends CommonSubtitlesParams {
+}
+interface DeleteSubtitlesParams {
+    is_attachment: boolean;
+    sdk_domain: string;
+    space_id: number;
+    record_id: number;
+    uuid?: string;
+    uuids: string[];
+}
+
+declare const addSubtitles: ({ dataSource, sdk_domain, ffmpegPath, id, debug, }: AddSubtitlesParams) => Promise<void>;
+declare const deleteSubtitles: (params: DeleteSubtitlesParams) => Promise<void>;
+
+export { addSubtitles, deleteSubtitles };

package/dist/main.d.ts

@@ -1,3 +1,24 @@
-import { AddSubtitlesParams, DeleteSubtitleParams } from './types';
-export declare const addSubtitle: ({ dataSource, sdk_domain, ffmpegPath, id, }: AddSubtitlesParams) => Promise<void>;
-export declare const deleteSubtitle: (params: DeleteSubtitleParams) => Promise<void>;
+import { DataSource } from 'typeorm';
+
+interface CommonSubtitlesParams {
+    ffmpegPath: string;
+    sdk_domain: string;
+    dataSource: DataSource;
+    id: number;
+    debug?: boolean;
+}
+interface AddSubtitlesParams extends CommonSubtitlesParams {
+}
+interface DeleteSubtitlesParams {
+    is_attachment: boolean;
+    sdk_domain: string;
+    space_id: number;
+    record_id: number;
+    uuid?: string;
+    uuids: string[];
+}
+
+declare const addSubtitles: ({ dataSource, sdk_domain, ffmpegPath, id, debug, }: AddSubtitlesParams) => Promise<void>;
+declare const deleteSubtitles: (params: DeleteSubtitlesParams) => Promise<void>;
+
+export { addSubtitles, deleteSubtitles };

package/dist/main.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.deleteSubtitle=exports.addSubtitle=void 0;const add_1=require("./add"),delete_1=require("./delete"),addSubtitle=async({dataSource:e,sdk_domain:t,ffmpegPath:d,id:a})=>{const i=new add_1.Add({dataSource:e,sdk_domain:t,ffmpegPath:d,id:a});await i.start()};exports.addSubtitle=addSubtitle;const deleteSubtitle=async e=>{const t=new delete_1.DeleteSubtitle(e);await t.start()};exports.deleteSubtitle=deleteSubtitle;
+var t,e=Object.create,i=Object.defineProperty,s=Object.getOwnPropertyDescriptor,a=Object.getOwnPropertyNames,o=Object.getPrototypeOf,d=Object.prototype.hasOwnProperty,l=(t,e,o,l)=>{if(e&&"object"==typeof e||"function"==typeof e)for(let r of a(e))d.call(t,r)||r===o||i(t,r,{get:()=>e[r],enumerable:!(l=s(e,r))||l.enumerable});return t},r={};((t,e)=>{for(var s in e)i(t,s,{get:e[s],enumerable:!0})})(r,{addSubtitles:()=>R,deleteSubtitles:()=>U}),module.exports=(t=r,l(i({},"__esModule",{value:!0}),t));var n=require("@soga/sdk"),u=((t,s,a)=>(a=null!=t?e(o(t)):{},l(!s&&t&&t.__esModule?a:i(a,"default",{value:t,enumerable:!0}),t)))(require("axios")),c=class{sdk_domain;sdk;constructor(t){this.sdk_domain=t,this.sdk=new n.Sdk(this.sdk_domain)}async deleteBaiduHostFile({fs_id:t,host_id:e}){const{access_token:i}=await this.getBaiduAuthData({host_id:e}),s=`https://pan.baidu.com/rest/2.0/xpan/multimedia?method=filemetas&access_token=${i}&fsids=%5B${t}%5D`,a=await u.default.get(s),{errno:o,list:d}=a.data;if(o)return;const l=d[0]?.path;if(l){const t=this.getSearchParams({async:"2",filelist:JSON.stringify([l])});await u.default.post(`https://pan.baidu.com/rest/2.0/xpan/file?method=filemanager&access_token=${i}&opera=delete`,t)}}async deleteAliHostFile({host_id:t,file_id:e}){const{access_token:i,drive_id:s}=await this.sdk.getAliHostAuthData({host_id:t});await u.default.post("https://openapi.alipan.com/adrive/v1.0/openFile/delete",{drive_id:s,file_id:e},{headers:{Authorization:`Bearer ${i}`}})}async getBaiduAuthData({host_id:t}){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:t,refresh:!1})).access_token}}getSearchParams(t){Object.keys(t).forEach(e=>{void 0===t[e]&&delete t[e]});return new URLSearchParams(t).toString()}},f=require("@soga/entities"),h=require("fs-extra"),_=require("@soga/types"),p=require("uuid"),m=require("path"),b=require("@soga/utils"),g=require("@soga/fileutils"),w=require("child_process"),y=require("@soga/fetcher"),S=require("crypto"),v=require("fs"),A=require("fs-extra"),z=require("path");async function k(t){await new Promise((e,i)=>{(async()=>{const s=(0,z.resolve)(t.outputRoot,t.filename);await(0,A.remove)(s);let a=!1;const o=(0,v.createWriteStream)(s);o.on("open",()=>{a=!0}).on("error",async t=>{a&&o.close(),await(0,A.remove)(s),i(t)});const{length:d}=t.files,l=async(e=0)=>{if(e<d){const i=t.files[e],d=t.files.length;await new Promise((t,e)=>{const l=(0,v.createReadStream)(i);l.pipe(o,{end:!d}),l.on("end",()=>{l.close(),t(!0)}),l.on("error",async t=>{l.destroy(),a&&o.close(),await(0,A.remove)(s),e(t)})}),await l(e+1)}};await l(0),a&&o.end(),e(!0)})()})}var F=class extends c{debug=!1;safe_file_name="safe.bin";dataSource;repository;queues=[];ffmpegPath;id;mediaSubtitles;old_detail;size_limits={baidu:52428800,ali:53687091200};hosts=[];indexes={};groups={};upload_results={};positions={};file_map={};constructor({dataSource:t,sdk_domain:e,ffmpegPath:i,id:s,debug:a}){super(e),this.id=s,this.dataSource=t,this.ffmpegPath=i,this.repository=t.getRepository(f.MediaSubtitles),this.debug=a||!1}async init(){const t=await this.repository.findOneBy({id:this.id});this.old_detail=t.record_info,this.mediaSubtitles=await this.repository.findOneBy({id:this.id}),await(0,h.remove)(this.mediaSubtitles.output_root),await(0,h.ensureDir)(this.mediaSubtitles.output_root),this.hosts=Object.keys(this.mediaSubtitles.record_info.cloud_info);for(const t of this.hosts)this.file_map[t]={},this.indexes[t]=0,this.groups[t]=[],this.positions[t]=0,this.upload_results[t]={}}safe_file_info;async insertSafeFile(t){return this.safe_file_info||(this.safe_file_info=await this.generateSafeFile()),{...this.safe_file_info,host_type:t}}async generateVtt(t){const e=(0,p.v4)(),{ext:i}=(0,m.parse)(t),{output_root:s}=this.mediaSubtitles,a=`${e}.vtt`,o=(0,m.resolve)(s,a),d=`${e}.zip`,l=(0,m.resolve)(s,d),r=`formatted_input${i}`,n=(0,m.resolve)(s,r),u=`${(0,p.v4)()}.zip`,c=(0,m.resolve)(s,u);if(await(0,b.isUtf8File)(t))await(0,h.copy)(t,n);else{await(0,b.saveFileAsUtf8)(t,n);if(!await(0,g.isValidFile)(n))return}const f=["-i",n,"-c","webvtt","-y",a];await this.ffmpeg(f,{cwd:s});if(await(0,g.isValidFile)(o)){const{label:e,lang:s}=await(0,b.getFileLanguage)(o);await(0,b.gzip)(t,c),await(0,b.gzip)(o,l);const a=await(0,g.getFileSize)(c);return{file:d,path:l,size:await(0,g.getFileSize)(l),lang:s,name:e,title:e||(0,m.parse)(t).name,builtin:!1,source:{file:u,path:c,size:a,ext:i}}}}async ffmpeg(t,e){const{ffmpegPath:i}=this;return await new Promise((s,a)=>{const o=(0,w.spawn)(i,t,Object.assign({stdio:"ignore"},e));o.on("close",t=>{s(t)}),o.on("error",t=>{a(t)})})}async downloadFromBaidu(){const{record_id:t,output_root:e,is_attachment:i,uuid:s}=this.mediaSubtitles,{manifest:a,cloud_info:o}=this.mediaSubtitles.record_info;let d,l;if(i){const t=a.attachments.find(t=>t.uuid===s);d=t.subtitles,l=t.baidu}else d=a.subtitles,l=a.baidu;if(!d?.length)return;if(!l)return;const{access_token:r}=await this.getBaiduAuthData();for(const i of d){const{uuid:s,name:a,lang:d,builtin:n,title:u}=i,c=l.media?.subtitles[s],{file:f,start:_,end:p,size:b,source:g}=c,w=l.files[f],{fs_id:S,md5:v}=w,A=o.baidu.id,z=o.baidu.vip_type,k=await(0,y.fetchBaiduBuffer)({dataSource:this.dataSource,record_id:t,start:_,end:p,total:b,access_token:r,host_id:A,host_vip_type:z,fs_id:S,part_md5:v,abort_controller:new AbortController}),F=(0,m.resolve)(e,s);await(0,h.writeFile)(F,k);const O={builtin:n,file:s,path:F,size:b,lang:d,name:a,title:u};if(g){const{start:i,end:a,size:o,ext:d}=g,{fs_id:l,md5:n}=w,u=await(0,y.fetchBaiduBuffer)({dataSource:this.dataSource,record_id:t,start:i,end:a,total:o,access_token:r,host_id:A,host_vip_type:z,fs_id:l,part_md5:n,abort_controller:new AbortController}),c=`${s}_source`,f=(0,m.resolve)(e,c);await(0,h.writeFile)(f,u),O.source={file:c,path:f,size:o,ext:d}}this.queues.push(O)}}async downloadFromAli(){const{record_id:t,output_root:e,is_attachment:i,uuid:s}=this.mediaSubtitles,{manifest:a,cloud_info:o}=this.mediaSubtitles.record_info;let d,l;if(i){const t=a.attachments.find(t=>t.uuid===s);d=t.subtitles,l=t.ali}else d=a.subtitles,l=a.ali;if(!d?.length)return;if(!l)return;const{access_token:r}=await this.getAliAuthData();for(const i of d){const{uuid:s,name:a,lang:d,builtin:n,title:u}=i,c=l.media?.subtitles[s],{file:f,start:_,end:p,size:b,source:g}=c,w=l.files[f],{file_id:S,md5:v}=w,A=o.ali.id,z=o.ali.vip_type,k=await(0,y.fetchAliBuffer)({dataSource:this.dataSource,record_id:t,start:_,end:p,total:b,access_token:r,host_id:A,host_vip_type:z,file_id:S,drive_id:o.ali.drive_id,part_md5:v,abort_controller:new AbortController}),F=(0,m.resolve)(e,s);await(0,h.writeFile)(F,k);const O={builtin:n,file:s,path:F,size:b,lang:d,name:a,title:u};if(g){const{file:i,start:a,end:d,size:l,ext:n}=g,{file_id:u,md5:c}=w,f=await(0,y.fetchAliBuffer)({dataSource:this.dataSource,record_id:t,start:a,end:d,total:l,access_token:r,host_id:A,host_vip_type:z,file_id:u,drive_id:o.ali.drive_id,part_md5:c,abort_controller:new AbortController}),_=`${s}_source`,p=(0,m.resolve)(e,_);await(0,h.writeFile)(p,f),O.source={file:_,path:p,size:l,ext:n}}this.queues.push(O)}}async getBaiduAuthData(){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[_.HostType.BAIDU].id,refresh:!1})).access_token}}async getAliAuthData(){const t=await this.sdk.getAliHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[_.HostType.ALI].id,refresh:!1});return{drive_id:t.drive_id,access_token:t.access_token}}async generateSafeFile(){const{record_info:t,output_root:e}=this.mediaSubtitles,{manifest:i}=t,s=function(t){const e=(0,S.createHash)("md5");return e.update(`${t}`),e.digest("hex")}(i.metas.file.size),a=(0,m.resolve)(e,this.safe_file_name);return await(0,h.writeFile)(a,s),{file_path:a,size:s.length}}async freshRecordInfo(){const{space_id:t,record_id:e}=this.mediaSubtitles;await this.sdk.getRecordInfo({space_id:t,record_id:e,refresh:!0})}},O=require("@soga/types"),H=require("path"),B=require("@soga/fileutils"),q=require("@soga/utils"),I=require("@soga/part-uploader"),D=require("fs-extra"),j=require("@soga/error"),x=require("uuid"),T=class extends F{async start(){try{await this.init(),await this.generateVtts(),await this.downloadFile(),await this.calculateSubtitles(),await this.processGroup(),await this.uploadFiles(),await this.updateRecordInfo(),this.debug?await(0,D.writeJSON)((0,H.resolve)(this.mediaSubtitles.output_root,"runtime.json"),{groups:this.groups,file_map:this.file_map,upload_results:this.upload_results,queues:this.queues}):await this.destroy()}catch(t){throw(0,j.buildDError)(t,{message:"Error occurred during add subtitles"})}}async destroy(){try{await this.repository.delete({id:this.id});const{output_root:t}=this.mediaSubtitles;await(0,D.remove)(t)}catch(t){}}async updateRecordInfo(){const{hosts:t,queues:e,mediaSubtitles:i,old_detail:s}=this,{space_id:a,record_id:o,is_attachment:d,uuid:l}=this.mediaSubtitles,{manifest:r}=i.record_info;if(!r)return;const n=[],u={baidu:{},ali:{}};let c=!0;for(const i of t){for(const t of e){const{file:e,path:s,builtin:a,lang:o,name:d,title:l,size:r,source:f}=t,h=this.file_map[i][e],_=e.split(".")[0];c&&n.push({builtin:a,lang:o,name:d,title:l,uuid:_});const p={file:h.group_file,start:h.start,end:h.end,size:h.size};if(f){const{file:t,ext:e}=f,s=this.file_map[i][t];p.source={file:s.group_file,start:s.start,end:s.end,size:s.size,ext:e}}u[i][_]=p}c=!1}if(d){const e=r.attachments?.find(t=>t.uuid==l);if(!e)return;e.subtitles=n;for(const i of t){const t=e[i];t?.media&&(t?.media?.subtitles&&Object.values(t.media.subtitles).forEach(e=>{const{file:i}=e;delete t.files[i]}),t.media.subtitles=u[i],t.files={...t.files,...this.upload_results[i]})}}else{r.subtitles=n;for(const e of t){const t=r[e];t?.media&&(t?.media?.subtitles&&(Object.values(t.media.subtitles).forEach(e=>{const{file:i}=e;delete t.files[i]}),t.media.subtitles=u[e],t.files={...t.files,...this.upload_results[e]}))}}r.hosts=r.hosts||{};for(const e of t){const t=(0,q.calculateManifestHostSize)(r,e);r.hosts[e]=t}if((await this.sdk.getRecordInfo({space_id:a,record_id:o,refresh:!0})).manifest_md5!=s.manifest_md5)throw(0,j.buildDError)(new Error("File has been modified."));await this.sdk.updateRecord({space_id:this.mediaSubtitles.space_id,record_id:this.mediaSubtitles.record_id,manifest:JSON.stringify(r),parent_id:this.mediaSubtitles.record_info.parent_id});try{await this.deleteOldSubtitleFiles(r)}catch(t){}}getKey(t,e){return`${t}_${e}`}async deleteOldSubtitleFiles(t){const{old_detail:e,hosts:i}=this,{is_attachment:s,uuid:a}=this.mediaSubtitles,{manifest:o}=e;if(!o)return;const d={baidu:[],ali:[]},l={};for(const e of i){const i=t[e]?.files||{},s=t.attachments||[];Object.values(i).forEach(t=>{if(e===O.HostType.BAIDU){const i=this.getKey(e,t.fs_id);l[i]=!0}else if(e===O.HostType.ALI){const i=this.getKey(e,t.file_id);l[i]=!0}}),Object.values(s).forEach(t=>{const i=t[e],s=i?.files||{};Object.values(s).forEach(t=>{if(e===O.HostType.BAIDU){const i=this.getKey(e,t.fs_id);l[i]=!0}else if(e===O.HostType.ALI){const i=this.getKey(e,t.file_id);l[i]=!0}})})}let r,n,u;if(s){const t=o.attachments?.find(t=>t.uuid===a);if(!t?.subtitles?.length)return;u=t.subtitles,r=t.baidu,n=t.ali}else{const{subtitles:t}=o;if(!t?.length)return;u=t,r=o.baidu,n=o.ali}if(r){const{files:t}=r;u.map(e=>{const{uuid:i}=e,s=r.media?.subtitles[i];if(!s)return null;const{file:a}=s;return t[a]}).filter(t=>!!t).forEach(t=>{const e=this.getKey(O.HostType.BAIDU,t.fs_id);l[e]||d.baidu.push({fs_id:t.fs_id})})}if(n){const{files:t}=n;u.map(e=>{const{uuid:i}=e,s=n.media?.subtitles[i];if(!s)return null;const{file:a}=s;return t[a]}).filter(t=>!!t).forEach(t=>{const e=this.getKey(O.HostType.ALI,t.file_id);l[e]||d.ali.push({file_id:t.file_id})})}if(r&&d.baidu.length){const t=this.mediaSubtitles.record_info.cloud_info.baidu.id;for(const e of d.baidu)await this.deleteBaiduHostFile({host_id:t,fs_id:e.fs_id})}if(n&&d.ali.length){const t=this.mediaSubtitles.record_info.cloud_info.ali.id;for(const e of d.ali)await this.deleteAliHostFile({host_id:t,file_id:e.file_id})}}async uploadFiles(){const{hosts:t,mediaSubtitles:e}=this;for(const e of t){const t=this.groups[e];let i=1;for(const s of t){const{filename:t}=s;e===O.HostType.BAIDU?await this.uploadBaidu(t,i++):e===O.HostType.ALI&&await this.uploadAli(t,i++)}}}async uploadBaidu(t,e){const{output_root:i,record_info:s}=this.mediaSubtitles,a=(0,H.resolve)(i,t),o=await(0,B.getFileSize)(a),d={file:a,start:0,end:o-1},l=await(0,q.calculateMd5)(d),r=await(0,q.calculateMd4)(d),{fs_id:n}=await(0,I.uploadBaidu)({file_id:1,part_id:e,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain,host_id:s.cloud_info.baidu.id,cloud_folder_path:s.cloud_info.baidu.path,part:{file:t,start:0,end:o-1,size:o,md5:l,md4:r,path:a},appid:30045106});this.upload_results.baidu[t]={md5:l,size:o,md4:r,fs_id:n,preview:!0,source:!1}}async uploadAli(t,e){const{output_root:i,record_info:s}=this.mediaSubtitles,a=(0,H.resolve)(i,t),o=await(0,B.getFileSize)(a),d={file:a,start:0,end:o-1},l=await(0,q.calculateMd5)(d),r=await(0,q.calculateSha1)(d),{file_id:n}=await(0,I.uploadAli)({file_id:1,part_id:e,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain,host_id:s.cloud_info.ali.id,cloud_folder_id:s.cloud_info.ali.file_id,part:{file:t,start:0,end:o-1,size:o,md5:l,path:a}});this.upload_results.ali[t]={md5:l,sha1:r,size:o,file_id:n,preview:!0,source:!1}}fileMap={};getFileName=(t,e)=>{const i=`sub_${t}_${e}`;if(this.fileMap[i])return this.fileMap[i];const s=`sub-${(0,x.v4)()}`;return this.fileMap[i]=s,s};addOne({file_path:t,size:e,host_type:i},s=!1){const a=this.indexes[i],o=this.positions[i],d=this.groups[i],l=this.size_limits[i];if(s&&d[a]||!(o+e<=l)){this.indexes[i]++;const s=this.indexes[i];d[s]?d[s].files.push(t):d[s]={filename:this.getFileName(i,s),files:[t]};const a=0,o=e-1;return this.positions[i]=e,{file_index:s,group_file:this.getFileName(i,s),start:a,end:o,size:o-a+1}}{const s=o,l=o+e-1;return this.positions[i]=l+1,d[a]?d[a].files.push(t):d[a]={filename:this.getFileName(i,a),files:[t]},{file_index:a,group_file:this.getFileName(i,a),start:s,end:l,size:l-s+1}}}async processGroup(){const{hosts:t}=this;for(const e of t){const t=this.groups[e];for(const e of t)await k({outputRoot:this.mediaSubtitles.output_root,files:e.files,filename:e.filename})}}async calculateSubtitles(){const{hosts:t}=this;for(const e of t){const{file_path:t,size:i}=await this.insertSafeFile(e);this.addOne({file_path:t,size:i,host_type:e},!0);for(const t of this.queues){const i=this.addOne({file_path:t.path,size:t.size,host_type:e});this.file_map[e][t.file]=i;const{source:s}=t;if(s){const t=this.addOne({file_path:s.path,size:s.size,host_type:e});this.file_map[e][s.file]=t}}}}async generateVtts(){const{inputs:t}=this.mediaSubtitles,{length:e}=t;for(let i=0;i<e;i++){const e=t[i],s=await this.generateVtt(e);s&&this.queues.push(s)}}async downloadFile(){const{record_info:t}=this.mediaSubtitles;if(!t?.cloud_info)return;const{hosts:e}=this;e.includes(O.HostType.BAIDU)?await this.downloadFromBaidu():e.includes(O.HostType.ALI)&&await this.downloadFromAli()}},E=require("@soga/types"),P=require("@soga/utils"),$=class extends c{space_id;record_id;uuid;uuids;detail;is_attachment;constructor({sdk_domain:t,space_id:e,record_id:i,uuid:s,uuids:a,is_attachment:o}){super(t),this.space_id=e,this.record_id=i,this.uuid=s,this.uuids=a,this.is_attachment=o}datas={baidu:{},ali:{}};async deleteHostData(t){const{uuid:e,uuids:i}=this,{manifest:s}=this.detail;let a,o;if(this.is_attachment){if(!s.attachments)return;if(o=s.attachments?.find(t=>t.uuid==e),!o?.subtitles)return;a=o[t]}else{if(!s.subtitles)return;if(a=s[t],!a?.media?.subtitles)return}const d=this.is_attachment?o.subtitles:s.subtitles;if(!d?.length)return;const l=d.filter(t=>this.uuids.includes(t.uuid));if(l?.length&&a.media?.subtitles){const{subtitles:e}=a.media,s={};for(const t of i){const i=e[t];if(!i)continue;const{file:a}=i;s[a]=!0,delete e[t]}Object.values(e).forEach(t=>{const{file:e}=t;delete s[e]});const o=Object.keys(s);if(o.length){const{files:e}=a;for(const i of o)if(t===E.HostType.BAIDU){const t=e[i];this.datas.baidu[t.fs_id]={file:i,fs_id:t.fs_id}}else if(t===E.HostType.ALI){const t=e[i];this.datas.ali[t.file_id]={file:i,file_id:t.file_id}}}Object.keys(s).forEach(t=>{delete a.files[t]}),this.validateAndDeleteFiles(t)}}validateAndDeleteFiles(t){const{manifest:e}=this.detail,i=e[t]?.files||{},s=e.attachments||[];Object.values(i).forEach(e=>{t===E.HostType.BAIDU?delete this.datas.baidu[e.fs_id]:t===E.HostType.ALI&&delete this.datas.ali[e.file_id]}),Object.values(s).forEach(e=>{const i=e[t],s=i?.files||{};Object.values(s).forEach(e=>{t===E.HostType.BAIDU?delete this.datas.baidu[e.fs_id]:t===E.HostType.ALI&&delete this.datas.ali[e.file_id]})});const a=Object.values(this.datas.baidu);if(0==Object.values(this.datas.ali).length&&0==a.length)return;const o=t=>{const i=Object.values(this.datas[t]);for(const s of i){const{file:i}=s;if(this.is_attachment){const s=e.attachments?.find(t=>t.uuid===this.uuid);if(s){const e=s[t];e&&e.files&&delete e.files[i]}}else{const s=e[t];s&&s.files&&delete s.files[i]}}};o(E.HostType.BAIDU),o(E.HostType.ALI)}async deleteFiles(){const t=Object.values(this.datas.baidu),e=Object.values(this.datas.ali);if(t.length){const e=this.detail.cloud_info.baidu.id;for(const i of t)await this.deleteBaiduHostFile({host_id:e,fs_id:i.fs_id})}if(e.length){const t=this.detail.cloud_info.ali.id;for(const i of e)await this.deleteAliHostFile({host_id:t,file_id:i.file_id})}}async start(){const{space_id:t,record_id:e}=this,i=await this.getRecordInfo();this.detail=i;const{manifest:s,parent_id:a}=i,{cloud_info:o}=i,d=Object.keys(o),{length:l}=d;if(l){for(let t=0;t<l;t++){const e=d[t];await this.deleteHostData(e)}if(this.is_attachment){const t=s.attachments?.find(t=>t.uuid==this.uuid);t.subtitles=t.subtitles.filter(t=>!this.uuids.includes(t.uuid))}else s.subtitles=s.subtitles.filter(t=>!this.uuids.includes(t.uuid));if(d.includes(E.HostType.BAIDU)){const t=(0,P.calculateManifestHostSize)(this.detail.manifest,E.HostType.BAIDU);s.baidu=s.baidu||{files:{},host_id:i.cloud_info.baidu.id,max_size:0},s.hosts=s.hosts||{},s.hosts.baidu=t}if(d.includes(E.HostType.ALI)){const t=(0,P.calculateManifestHostSize)(this.detail.manifest,E.HostType.ALI);s.ali=s.ali||{drive_id:i.cloud_info.ali.drive_id,files:{},host_id:i.cloud_info.ali.id,max_size:0},s.hosts=s.hosts||{},s.hosts.ali=t}await this.sdk.updateRecord({space_id:t,record_id:e,manifest:JSON.stringify(s),parent_id:a});try{await this.deleteFiles()}catch(t){}}}async getRecordInfo(t=!0){return await this.sdk.getRecordInfo({space_id:this.space_id,record_id:this.record_id,refresh:t})}},R=async({dataSource:t,sdk_domain:e,ffmpegPath:i,id:s,debug:a})=>{const o=new T({dataSource:t,sdk_domain:e,ffmpegPath:i,id:s,debug:a});await o.start()},U=async t=>{const e=new $(t);await e.start()};

package/dist/main.mjs

@@ -0,0 +1 @@
+import{Sdk as t}from"@soga/sdk";import i from"axios";var e=class{sdk_domain;sdk;constructor(i){this.sdk_domain=i,this.sdk=new t(this.sdk_domain)}async deleteBaiduHostFile({fs_id:t,host_id:e}){const{access_token:s}=await this.getBaiduAuthData({host_id:e}),a=`https://pan.baidu.com/rest/2.0/xpan/multimedia?method=filemetas&access_token=${s}&fsids=%5B${t}%5D`,o=await i.get(a),{errno:d,list:n}=o.data;if(d)return;const l=n[0]?.path;if(l){const t=this.getSearchParams({async:"2",filelist:JSON.stringify([l])});await i.post(`https://pan.baidu.com/rest/2.0/xpan/file?method=filemanager&access_token=${s}&opera=delete`,t)}}async deleteAliHostFile({host_id:t,file_id:e}){const{access_token:s,drive_id:a}=await this.sdk.getAliHostAuthData({host_id:t});await i.post("https://openapi.alipan.com/adrive/v1.0/openFile/delete",{drive_id:a,file_id:e},{headers:{Authorization:`Bearer ${s}`}})}async getBaiduAuthData({host_id:t}){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:t,refresh:!1})).access_token}}getSearchParams(t){Object.keys(t).forEach(i=>{void 0===t[i]&&delete t[i]});return new URLSearchParams(t).toString()}};import{MediaSubtitles as s}from"@soga/entities";import{copy as a,ensureDir as o,remove as d,writeFile as n}from"fs-extra";import{HostType as l}from"@soga/types";import{v4 as r}from"uuid";import{parse as u,resolve as c}from"path";import{getFileLanguage as f,isUtf8File as h,saveFileAsUtf8 as _,gzip as m}from"@soga/utils";import{getFileSize as p,isValidFile as b}from"@soga/fileutils";import{spawn as g}from"child_process";import{fetchAliBuffer as w,fetchBaiduBuffer as y}from"@soga/fetcher";import{createHash as S}from"crypto";import{createReadStream as A,createWriteStream as v}from"fs";import{remove as k}from"fs-extra";import{resolve as z}from"path";async function F(t){await new Promise((i,e)=>{(async()=>{const s=z(t.outputRoot,t.filename);await k(s);let a=!1;const o=v(s);o.on("open",()=>{a=!0}).on("error",async t=>{a&&o.close(),await k(s),e(t)});const{length:d}=t.files,n=async(i=0)=>{if(i<d){const e=t.files[i],d=t.files.length;await new Promise((t,i)=>{const n=A(e);n.pipe(o,{end:!d}),n.on("end",()=>{n.close(),t(!0)}),n.on("error",async t=>{n.destroy(),a&&o.close(),await k(s),i(t)})}),await n(i+1)}};await n(0),a&&o.end(),i(!0)})()})}var I=class extends e{debug=!1;safe_file_name="safe.bin";dataSource;repository;queues=[];ffmpegPath;id;mediaSubtitles;old_detail;size_limits={baidu:52428800,ali:53687091200};hosts=[];indexes={};groups={};upload_results={};positions={};file_map={};constructor({dataSource:t,sdk_domain:i,ffmpegPath:e,id:a,debug:o}){super(i),this.id=a,this.dataSource=t,this.ffmpegPath=e,this.repository=t.getRepository(s),this.debug=o||!1}async init(){const t=await this.repository.findOneBy({id:this.id});this.old_detail=t.record_info,this.mediaSubtitles=await this.repository.findOneBy({id:this.id}),await d(this.mediaSubtitles.output_root),await o(this.mediaSubtitles.output_root),this.hosts=Object.keys(this.mediaSubtitles.record_info.cloud_info);for(const t of this.hosts)this.file_map[t]={},this.indexes[t]=0,this.groups[t]=[],this.positions[t]=0,this.upload_results[t]={}}safe_file_info;async insertSafeFile(t){return this.safe_file_info||(this.safe_file_info=await this.generateSafeFile()),{...this.safe_file_info,host_type:t}}async generateVtt(t){const i=r(),{ext:e}=u(t),{output_root:s}=this.mediaSubtitles,o=`${i}.vtt`,d=c(s,o),n=`${i}.zip`,l=c(s,n),g=c(s,`formatted_input${e}`),w=`${r()}.zip`,y=c(s,w);if(await h(t))await a(t,g);else{await _(t,g);if(!await b(g))return}const S=["-i",g,"-c","webvtt","-y",o];await this.ffmpeg(S,{cwd:s});if(await b(d)){const{label:i,lang:s}=await f(d);await m(t,y),await m(d,l);const a=await p(y);return{file:n,path:l,size:await p(l),lang:s,name:i,title:i||u(t).name,builtin:!1,source:{file:w,path:y,size:a,ext:e}}}}async ffmpeg(t,i){const{ffmpegPath:e}=this;return await new Promise((s,a)=>{const o=g(e,t,Object.assign({stdio:"ignore"},i));o.on("close",t=>{s(t)}),o.on("error",t=>{a(t)})})}async downloadFromBaidu(){const{record_id:t,output_root:i,is_attachment:e,uuid:s}=this.mediaSubtitles,{manifest:a,cloud_info:o}=this.mediaSubtitles.record_info;let d,l;if(e){const t=a.attachments.find(t=>t.uuid===s);d=t.subtitles,l=t.baidu}else d=a.subtitles,l=a.baidu;if(!d?.length)return;if(!l)return;const{access_token:r}=await this.getBaiduAuthData();for(const e of d){const{uuid:s,name:a,lang:d,builtin:u,title:f}=e,h=l.media?.subtitles[s],{file:_,start:m,end:p,size:b,source:g}=h,w=l.files[_],{fs_id:S,md5:A}=w,v=o.baidu.id,k=o.baidu.vip_type,z=await y({dataSource:this.dataSource,record_id:t,start:m,end:p,total:b,access_token:r,host_id:v,host_vip_type:k,fs_id:S,part_md5:A,abort_controller:new AbortController}),F=c(i,s);await n(F,z);const I={builtin:u,file:s,path:F,size:b,lang:d,name:a,title:f};if(g){const{start:e,end:a,size:o,ext:d}=g,{fs_id:l,md5:u}=w,f=await y({dataSource:this.dataSource,record_id:t,start:e,end:a,total:o,access_token:r,host_id:v,host_vip_type:k,fs_id:l,part_md5:u,abort_controller:new AbortController}),h=`${s}_source`,_=c(i,h);await n(_,f),I.source={file:h,path:_,size:o,ext:d}}this.queues.push(I)}}async downloadFromAli(){const{record_id:t,output_root:i,is_attachment:e,uuid:s}=this.mediaSubtitles,{manifest:a,cloud_info:o}=this.mediaSubtitles.record_info;let d,l;if(e){const t=a.attachments.find(t=>t.uuid===s);d=t.subtitles,l=t.ali}else d=a.subtitles,l=a.ali;if(!d?.length)return;if(!l)return;const{access_token:r}=await this.getAliAuthData();for(const e of d){const{uuid:s,name:a,lang:d,builtin:u,title:f}=e,h=l.media?.subtitles[s],{file:_,start:m,end:p,size:b,source:g}=h,y=l.files[_],{file_id:S,md5:A}=y,v=o.ali.id,k=o.ali.vip_type,z=await w({dataSource:this.dataSource,record_id:t,start:m,end:p,total:b,access_token:r,host_id:v,host_vip_type:k,file_id:S,drive_id:o.ali.drive_id,part_md5:A,abort_controller:new AbortController}),F=c(i,s);await n(F,z);const I={builtin:u,file:s,path:F,size:b,lang:d,name:a,title:f};if(g){const{file:e,start:a,end:d,size:l,ext:u}=g,{file_id:f,md5:h}=y,_=await w({dataSource:this.dataSource,record_id:t,start:a,end:d,total:l,access_token:r,host_id:v,host_vip_type:k,file_id:f,drive_id:o.ali.drive_id,part_md5:h,abort_controller:new AbortController}),m=`${s}_source`,p=c(i,m);await n(p,_),I.source={file:m,path:p,size:l,ext:u}}this.queues.push(I)}}async getBaiduAuthData(){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[l.BAIDU].id,refresh:!1})).access_token}}async getAliAuthData(){const t=await this.sdk.getAliHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[l.ALI].id,refresh:!1});return{drive_id:t.drive_id,access_token:t.access_token}}async generateSafeFile(){const{record_info:t,output_root:i}=this.mediaSubtitles,{manifest:e}=t,s=function(t){const i=S("md5");return i.update(`${t}`),i.digest("hex")}(e.metas.file.size),a=c(i,this.safe_file_name);return await n(a,s),{file_path:a,size:s.length}}async freshRecordInfo(){const{space_id:t,record_id:i}=this.mediaSubtitles;await this.sdk.getRecordInfo({space_id:t,record_id:i,refresh:!0})}};import{HostType as O}from"@soga/types";import{resolve as B}from"path";import{getFileSize as x}from"@soga/fileutils";import{calculateManifestHostSize as D,calculateMd4 as j,calculateMd5 as $,calculateSha1 as E}from"@soga/utils";import{uploadAli as H,uploadBaidu as R}from"@soga/part-uploader";import{remove as L,writeJSON as U}from"fs-extra";import{buildDError as P}from"@soga/error";import{v4 as q}from"uuid";var N=class extends I{async start(){try{await this.init(),await this.generateVtts(),await this.downloadFile(),await this.calculateSubtitles(),await this.processGroup(),await this.uploadFiles(),await this.updateRecordInfo(),this.debug?await U(B(this.mediaSubtitles.output_root,"runtime.json"),{groups:this.groups,file_map:this.file_map,upload_results:this.upload_results,queues:this.queues}):await this.destroy()}catch(t){throw P(t,{message:"Error occurred during add subtitles"})}}async destroy(){try{await this.repository.delete({id:this.id});const{output_root:t}=this.mediaSubtitles;await L(t)}catch(t){}}async updateRecordInfo(){const{hosts:t,queues:i,mediaSubtitles:e,old_detail:s}=this,{space_id:a,record_id:o,is_attachment:d,uuid:n}=this.mediaSubtitles,{manifest:l}=e.record_info;if(!l)return;const r=[],u={baidu:{},ali:{}};let c=!0;for(const e of t){for(const t of i){const{file:i,path:s,builtin:a,lang:o,name:d,title:n,size:l,source:f}=t,h=this.file_map[e][i],_=i.split(".")[0];c&&r.push({builtin:a,lang:o,name:d,title:n,uuid:_});const m={file:h.group_file,start:h.start,end:h.end,size:h.size};if(f){const{file:t,ext:i}=f,s=this.file_map[e][t];m.source={file:s.group_file,start:s.start,end:s.end,size:s.size,ext:i}}u[e][_]=m}c=!1}if(d){const i=l.attachments?.find(t=>t.uuid==n);if(!i)return;i.subtitles=r;for(const e of t){const t=i[e];t?.media&&(t?.media?.subtitles&&Object.values(t.media.subtitles).forEach(i=>{const{file:e}=i;delete t.files[e]}),t.media.subtitles=u[e],t.files={...t.files,...this.upload_results[e]})}}else{l.subtitles=r;for(const i of t){const t=l[i];t?.media&&(t?.media?.subtitles&&(Object.values(t.media.subtitles).forEach(i=>{const{file:e}=i;delete t.files[e]}),t.media.subtitles=u[i],t.files={...t.files,...this.upload_results[i]}))}}l.hosts=l.hosts||{};for(const i of t){const t=D(l,i);l.hosts[i]=t}if((await this.sdk.getRecordInfo({space_id:a,record_id:o,refresh:!0})).manifest_md5!=s.manifest_md5)throw P(new Error("File has been modified."));await this.sdk.updateRecord({space_id:this.mediaSubtitles.space_id,record_id:this.mediaSubtitles.record_id,manifest:JSON.stringify(l),parent_id:this.mediaSubtitles.record_info.parent_id});try{await this.deleteOldSubtitleFiles(l)}catch(t){}}getKey(t,i){return`${t}_${i}`}async deleteOldSubtitleFiles(t){const{old_detail:i,hosts:e}=this,{is_attachment:s,uuid:a}=this.mediaSubtitles,{manifest:o}=i;if(!o)return;const d={baidu:[],ali:[]},n={};for(const i of e){const e=t[i]?.files||{},s=t.attachments||[];Object.values(e).forEach(t=>{if(i===O.BAIDU){const e=this.getKey(i,t.fs_id);n[e]=!0}else if(i===O.ALI){const e=this.getKey(i,t.file_id);n[e]=!0}}),Object.values(s).forEach(t=>{const e=t[i],s=e?.files||{};Object.values(s).forEach(t=>{if(i===O.BAIDU){const e=this.getKey(i,t.fs_id);n[e]=!0}else if(i===O.ALI){const e=this.getKey(i,t.file_id);n[e]=!0}})})}let l,r,u;if(s){const t=o.attachments?.find(t=>t.uuid===a);if(!t?.subtitles?.length)return;u=t.subtitles,l=t.baidu,r=t.ali}else{const{subtitles:t}=o;if(!t?.length)return;u=t,l=o.baidu,r=o.ali}if(l){const{files:t}=l;u.map(i=>{const{uuid:e}=i,s=l.media?.subtitles[e];if(!s)return null;const{file:a}=s;return t[a]}).filter(t=>!!t).forEach(t=>{const i=this.getKey(O.BAIDU,t.fs_id);n[i]||d.baidu.push({fs_id:t.fs_id})})}if(r){const{files:t}=r;u.map(i=>{const{uuid:e}=i,s=r.media?.subtitles[e];if(!s)return null;const{file:a}=s;return t[a]}).filter(t=>!!t).forEach(t=>{const i=this.getKey(O.ALI,t.file_id);n[i]||d.ali.push({file_id:t.file_id})})}if(l&&d.baidu.length){const t=this.mediaSubtitles.record_info.cloud_info.baidu.id;for(const i of d.baidu)await this.deleteBaiduHostFile({host_id:t,fs_id:i.fs_id})}if(r&&d.ali.length){const t=this.mediaSubtitles.record_info.cloud_info.ali.id;for(const i of d.ali)await this.deleteAliHostFile({host_id:t,file_id:i.file_id})}}async uploadFiles(){const{hosts:t,mediaSubtitles:i}=this;for(const i of t){const t=this.groups[i];let e=1;for(const s of t){const{filename:t}=s;i===O.BAIDU?await this.uploadBaidu(t,e++):i===O.ALI&&await this.uploadAli(t,e++)}}}async uploadBaidu(t,i){const{output_root:e,record_info:s}=this.mediaSubtitles,a=B(e,t),o=await x(a),d={file:a,start:0,end:o-1},n=await $(d),l=await j(d),{fs_id:r}=await R({file_id:1,part_id:i,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain,host_id:s.cloud_info.baidu.id,cloud_folder_path:s.cloud_info.baidu.path,part:{file:t,start:0,end:o-1,size:o,md5:n,md4:l,path:a},appid:30045106});this.upload_results.baidu[t]={md5:n,size:o,md4:l,fs_id:r,preview:!0,source:!1}}async uploadAli(t,i){const{output_root:e,record_info:s}=this.mediaSubtitles,a=B(e,t),o=await x(a),d={file:a,start:0,end:o-1},n=await $(d),l=await E(d),{file_id:r}=await H({file_id:1,part_id:i,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain,host_id:s.cloud_info.ali.id,cloud_folder_id:s.cloud_info.ali.file_id,part:{file:t,start:0,end:o-1,size:o,md5:n,path:a}});this.upload_results.ali[t]={md5:n,sha1:l,size:o,file_id:r,preview:!0,source:!1}}fileMap={};getFileName=(t,i)=>{const e=`sub_${t}_${i}`;if(this.fileMap[e])return this.fileMap[e];const s=`sub-${q()}`;return this.fileMap[e]=s,s};addOne({file_path:t,size:i,host_type:e},s=!1){const a=this.indexes[e],o=this.positions[e],d=this.groups[e],n=this.size_limits[e];if(s&&d[a]||!(o+i<=n)){this.indexes[e]++;const s=this.indexes[e];d[s]?d[s].files.push(t):d[s]={filename:this.getFileName(e,s),files:[t]};const a=0,o=i-1;return this.positions[e]=i,{file_index:s,group_file:this.getFileName(e,s),start:a,end:o,size:o-a+1}}{const s=o,n=o+i-1;return this.positions[e]=n+1,d[a]?d[a].files.push(t):d[a]={filename:this.getFileName(e,a),files:[t]},{file_index:a,group_file:this.getFileName(e,a),start:s,end:n,size:n-s+1}}}async processGroup(){const{hosts:t}=this;for(const i of t){const t=this.groups[i];for(const i of t)await F({outputRoot:this.mediaSubtitles.output_root,files:i.files,filename:i.filename})}}async calculateSubtitles(){const{hosts:t}=this;for(const i of t){const{file_path:t,size:e}=await this.insertSafeFile(i);this.addOne({file_path:t,size:e,host_type:i},!0);for(const t of this.queues){const e=this.addOne({file_path:t.path,size:t.size,host_type:i});this.file_map[i][t.file]=e;const{source:s}=t;if(s){const t=this.addOne({file_path:s.path,size:s.size,host_type:i});this.file_map[i][s.file]=t}}}}async generateVtts(){const{inputs:t}=this.mediaSubtitles,{length:i}=t;for(let e=0;e<i;e++){const i=t[e],s=await this.generateVtt(i);s&&this.queues.push(s)}}async downloadFile(){const{record_info:t}=this.mediaSubtitles;if(!t?.cloud_info)return;const{hosts:i}=this;i.includes(O.BAIDU)?await this.downloadFromBaidu():i.includes(O.ALI)&&await this.downloadFromAli()}};import{HostType as K}from"@soga/types";import{calculateManifestHostSize as M}from"@soga/utils";var C=class extends e{space_id;record_id;uuid;uuids;detail;is_attachment;constructor({sdk_domain:t,space_id:i,record_id:e,uuid:s,uuids:a,is_attachment:o}){super(t),this.space_id=i,this.record_id=e,this.uuid=s,this.uuids=a,this.is_attachment=o}datas={baidu:{},ali:{}};async deleteHostData(t){const{uuid:i,uuids:e}=this,{manifest:s}=this.detail;let a,o;if(this.is_attachment){if(!s.attachments)return;if(o=s.attachments?.find(t=>t.uuid==i),!o?.subtitles)return;a=o[t]}else{if(!s.subtitles)return;if(a=s[t],!a?.media?.subtitles)return}const d=this.is_attachment?o.subtitles:s.subtitles;if(!d?.length)return;const n=d.filter(t=>this.uuids.includes(t.uuid));if(n?.length&&a.media?.subtitles){const{subtitles:i}=a.media,s={};for(const t of e){const e=i[t];if(!e)continue;const{file:a}=e;s[a]=!0,delete i[t]}Object.values(i).forEach(t=>{const{file:i}=t;delete s[i]});const o=Object.keys(s);if(o.length){const{files:i}=a;for(const e of o)if(t===K.BAIDU){const t=i[e];this.datas.baidu[t.fs_id]={file:e,fs_id:t.fs_id}}else if(t===K.ALI){const t=i[e];this.datas.ali[t.file_id]={file:e,file_id:t.file_id}}}Object.keys(s).forEach(t=>{delete a.files[t]}),this.validateAndDeleteFiles(t)}}validateAndDeleteFiles(t){const{manifest:i}=this.detail,e=i[t]?.files||{},s=i.attachments||[];Object.values(e).forEach(i=>{t===K.BAIDU?delete this.datas.baidu[i.fs_id]:t===K.ALI&&delete this.datas.ali[i.file_id]}),Object.values(s).forEach(i=>{const e=i[t],s=e?.files||{};Object.values(s).forEach(i=>{t===K.BAIDU?delete this.datas.baidu[i.fs_id]:t===K.ALI&&delete this.datas.ali[i.file_id]})});const a=Object.values(this.datas.baidu);if(0==Object.values(this.datas.ali).length&&0==a.length)return;const o=t=>{const e=Object.values(this.datas[t]);for(const s of e){const{file:e}=s;if(this.is_attachment){const s=i.attachments?.find(t=>t.uuid===this.uuid);if(s){const i=s[t];i&&i.files&&delete i.files[e]}}else{const s=i[t];s&&s.files&&delete s.files[e]}}};o(K.BAIDU),o(K.ALI)}async deleteFiles(){const t=Object.values(this.datas.baidu),i=Object.values(this.datas.ali);if(t.length){const i=this.detail.cloud_info.baidu.id;for(const e of t)await this.deleteBaiduHostFile({host_id:i,fs_id:e.fs_id})}if(i.length){const t=this.detail.cloud_info.ali.id;for(const e of i)await this.deleteAliHostFile({host_id:t,file_id:e.file_id})}}async start(){const{space_id:t,record_id:i}=this,e=await this.getRecordInfo();this.detail=e;const{manifest:s,parent_id:a}=e,{cloud_info:o}=e,d=Object.keys(o),{length:n}=d;if(n){for(let t=0;t<n;t++){const i=d[t];await this.deleteHostData(i)}if(this.is_attachment){const t=s.attachments?.find(t=>t.uuid==this.uuid);t.subtitles=t.subtitles.filter(t=>!this.uuids.includes(t.uuid))}else s.subtitles=s.subtitles.filter(t=>!this.uuids.includes(t.uuid));if(d.includes(K.BAIDU)){const t=M(this.detail.manifest,K.BAIDU);s.baidu=s.baidu||{files:{},host_id:e.cloud_info.baidu.id,max_size:0},s.hosts=s.hosts||{},s.hosts.baidu=t}if(d.includes(K.ALI)){const t=M(this.detail.manifest,K.ALI);s.ali=s.ali||{drive_id:e.cloud_info.ali.drive_id,files:{},host_id:e.cloud_info.ali.id,max_size:0},s.hosts=s.hosts||{},s.hosts.ali=t}await this.sdk.updateRecord({space_id:t,record_id:i,manifest:JSON.stringify(s),parent_id:a});try{await this.deleteFiles()}catch(t){}}}async getRecordInfo(t=!0){return await this.sdk.getRecordInfo({space_id:this.space_id,record_id:this.record_id,refresh:t})}},V=async({dataSource:t,sdk_domain:i,ffmpegPath:e,id:s,debug:a})=>{const o=new N({dataSource:t,sdk_domain:i,ffmpegPath:e,id:s,debug:a});await o.start()},J=async t=>{const i=new C(t);await i.start()};export{V as addSubtitles,J as deleteSubtitles};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/subtitles",
-  "version": "0.3.0",
+  "version": "1.0.0",
   "publishConfig": {
     "access": "public"
   },
@@ -10,26 +10,35 @@
   "files": [
     "dist"
   ],
+  "scripts": {
+    "demo": "tsx ./demo/demo.ts",
+    "build": "rimraf dist && tsup src/main.ts --format cjs,esm --dts --minify terser",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^11.0.4",
+    "@soga/typescript-config": "latest",
+    "rimraf": "^6.0.1",
+    "terser": "^5.43.1",
+    "tsup": "^8.5.0",
+    "tsx": "^4.20.6",
+    "typescript": "^5.8.3"
+  },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@soga/entities": "^0.3.0",
-    "@soga/fetcher": "^0.3.0",
-    "@soga/part-uploader": "^0.3.0",
-    "@soga/sdk": "^0.3.0",
-    "@soga/types": "^0.3.0",
-    "@soga/utils": "^0.3.0",
+    "@soga/entities": "latest",
+    "@soga/fetcher": "latest",
+    "@soga/part-uploader": "latest",
+    "@soga/sdk": "latest",
+    "@soga/types": "latest",
+    "@soga/utils": "latest",
+    "@soga/fileutils": "latest",
+    "@soga/error": "latest",
     "axios": "^1.9.0",
+    "fs-extra": "^11.3.2",
     "typeorm": "^0.3.24",
     "uuid": "^11.1.0"
-  },
-  "scripts": {
-    "build": "rimraf dist && tsc && ts-node ./scripts/minify",
-    "minify": "ts-node watch ./scripts/minify",
-    "demo": "ts-node  ./demo/demo.ts",
-    "test": "jest",
-    "dev": "tsc --watch",
-    "lint": "eslint . --ext .ts"
   }
-}
+}

package/dist/add.d.ts

@@ -1,93 +0,0 @@
-import { AddSubtitlesParams } from './types';
-import { RecordDetail } from '@soga/types';
-import { SpawnOptions } from 'child_process';
-import { DataSource, Repository } from 'typeorm';
-import { MediaSubtitles } from '@soga/entities';
-import { Common } from './common';
-export declare class Add extends Common {
-    protected dataSource: DataSource;
-    protected repository: Repository<MediaSubtitles>;
-    protected readonly ffmpegPath: string;
-    protected readonly id: number;
-    protected preview_file_name: string;
-    protected old_bin_name: string;
-    protected old_detail: RecordDetail;
-    protected get old_exist(): boolean;
-    protected mediaSubtitles: MediaSubtitles;
-    protected texts: any[];
-    protected segments: {
-        file: string;
-        start: number;
-        end: number;
-        size: number;
-    }[];
-    private vtt_infos;
-    constructor({ dataSource, sdk_domain, ffmpegPath, id }: AddSubtitlesParams);
-    start(): Promise<{
-        state: boolean;
-        message: any;
-    }>;
-    init(): Promise<void>;
-    downloadFile(): Promise<void>;
-    generateVtts(): Promise<void>;
-    generateBin(): Promise<void>;
-    uploadFile(): Promise<void>;
-    updateRecordInfo(): Promise<void>;
-    deleteOldFiles(): Promise<void>;
-    destroy(): Promise<void>;
-    protected freshRecordInfo(): Promise<void>;
-    protected getBaiduAuthData(): Promise<{
-        access_token: string;
-    }>;
-    protected getAliAuthData(): Promise<{
-        drive_id: string;
-        access_token: string;
-    }>;
-    protected ffmpeg(args: string[], options?: SpawnOptions): Promise<unknown>;
-    protected initPreviewFileName(): Promise<void>;
-    private calculateData;
-    private generateVtt;
-    private generateSafeFile;
-}
-export type VideoSubtitleData = {
-    file: string;
-    size: number;
-    lang: string;
-    name: string;
-    title: string;
-    builtin: boolean;
-    source: null | {
-        file: string;
-        start: number;
-        end: number;
-        size: number;
-        ext: string;
-    };
-};
-export type VideoSubtitleInfo = {
-    file: string;
-    size: number;
-    lang: string;
-    name: string;
-    title: string;
-    builtin: boolean;
-    bin: {
-        file: string;
-        start: number;
-        end: number;
-        size: number;
-    };
-    source: null | {
-        file: string;
-        start: number;
-        end: number;
-        size: number;
-        ext: string;
-        bin: {
-            file: string;
-            start: number;
-            end: number;
-            size: number;
-        };
-    };
-};

package/dist/add.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Add=void 0;const utils_1=require("@soga/utils"),path_1=require("path"),uuid_1=require("uuid"),utils_2=require("./utils"),types_1=require("@soga/types"),child_process_1=require("child_process"),fs_extra_1=require("fs-extra"),fetcher_1=require("@soga/fetcher"),entities_1=require("@soga/entities"),part_uploader_1=require("@soga/part-uploader"),utils_3=require("@soga/utils"),common_1=require("./common");class Add extends common_1.Common{dataSource;repository;ffmpegPath;id;preview_file_name;old_bin_name="old.bin";old_detail;get old_exist(){return!!this.old_detail?.manifest?.media?.texts?.length}mediaSubtitles;texts=[];segments=[];vtt_infos=[];constructor({dataSource:e,sdk_domain:t,ffmpegPath:i,id:s}){super(t),this.id=s,this.dataSource=e,this.ffmpegPath=i,this.repository=e.getRepository(entities_1.MediaSubtitles)}async start(){try{return await this.init(),await this.downloadFile(),await this.generateVtts(),await this.generateBin(),await this.uploadFile(),await this.updateRecordInfo(),await this.deleteOldFiles(),await this.destroy(),{state:!0,message:"Subtitles added successfully."}}catch(e){return{state:!1,message:e.message}}}async init(){const e=await this.repository.findOneBy({id:this.id});this.old_detail=e.record_info,this.mediaSubtitles=await this.repository.findOneBy({id:this.id}),await this.initPreviewFileName(),await(0,fs_extra_1.ensureDir)(this.mediaSubtitles.output_root)}async downloadFile(){if(!this.old_exist)return;const{record_info:e,record_id:t,output_root:i}=this.mediaSubtitles;if(!e?.cloud_info)return;if(!e?.manifest?.media?.texts?.length)return;const{manifest:s}=e,a=e.cloud_info[types_1.HostType.BAIDU];e.cloud_info[types_1.HostType.ALI];if(a){const e=s.parts[this.preview_file_name],{access_token:r}=await this.getBaiduAuthData(),o=this.dataSource,d=o.getRepository(entities_1.FetchUrl),l=(await d.find({}),await(0,fetcher_1.fetchBaiduBuffer)({dataSource:o,record_id:t,start:0,end:e.size-1,total:e.size,access_token:r,host_id:a.id,host_vip_type:a.vip_type,fs_id:s.baidu.info[this.preview_file_name]?.fs_id,part_md5:e.md5,abort_controller:new AbortController})),n=(0,path_1.resolve)(i,this.old_bin_name);await(0,fs_extra_1.writeFile)(n,l)}}async generateVtts(){const{inputs:e}=this.mediaSubtitles,{length:t}=e;for(let i=0;i<t;i++){const t=e[i],s=await this.generateVtt(t);s&&this.vtt_infos.push(s)}}async generateBin(){this.old_exist||await this.generateSafeFile(),this.texts=this.calculateData();const e=(0,path_1.resolve)(this.mediaSubtitles.output_root,this.preview_file_name);await(0,utils_2.mergeFileFragments)(this.segments,e)}async uploadFile(){const{cloud_info:e}=this.mediaSubtitles.record_info;if(!e)return;const t=(0,path_1.resolve)(this.mediaSubtitles.output_root,this.preview_file_name),i=await(0,utils_3.getFileSize)(t),s=await(0,utils_3.calculateMd5)({file:t,start:0,end:i-1}),a={file_id:1,part_id:1,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain},r={file:this.preview_file_name,start:0,end:i-1,size:i,path:t,index:0,md5:s};if(e.baidu){const o=await(0,utils_3.calculateMd4)({file:t,start:0,end:i-1}),{fs_id:d}=await(0,part_uploader_1.uploadBaidu)({...a,host_id:e.baidu.id,cloud_folder_path:e.baidu.path,part:{...r,md4:o}});await this.repository.update(this.id,{baidu_upload_result:{md5:s,size:i,md4:o,fs_id:d}})}if(e.ali){const o=await(0,utils_3.calculateSha1)({file:t,start:0,end:i-1}),{file_id:d}=await(0,part_uploader_1.uploadAli)({...a,host_id:e.ali.id,cloud_folder_id:e.ali.file_id,part:{...r,sha1:o}});await this.repository.update(this.id,{ali_upload_result:{md5:s,size:i,sha1:o,file_id:d}})}this.mediaSubtitles=await this.repository.findOneBy({id:this.id})}async updateRecordInfo(){const e=this.preview_file_name,{space_id:t,record_id:i,record_info:s,baidu_upload_result:a,ali_upload_result:r,output_root:o}=this.mediaSubtitles,{manifest:d,manifest_md5:l}=s;if(!s)return;const n=(0,path_1.resolve)(o,this.preview_file_name),_=await(0,utils_3.getFileSize)(n),u=await(0,utils_3.calculateMd5)({file:n,start:0,end:_-1});this.old_exist?(d.parts[e].md5=u,d.parts[e].size=_):(d.parts[e]={md5:u,preview:!0,size:_,source:!1},d.media.parts?.length?d.media.parts.push(e):d.media.parts=[e]),a&&(d.baidu.info[e]={fs_id:a.fs_id,md4:a.md4}),r&&(d.ali.info[e]={file_id:r.file_id,sha1:r.sha1}),d.media.texts=this.texts;let h=0;Object.values(d.parts).forEach((e=>{h+=e.size})),d.meta.host_size=h;if((await this.sdk.getRecordInfo({space_id:t,record_id:i,refresh:!0})).manifest_md5!=l)throw new Error("File has been modified.");await this.sdk.updateRecord({space_id:this.mediaSubtitles.space_id,record_id:this.mediaSubtitles.record_id,manifest:JSON.stringify(d),parent_id:this.mediaSubtitles.record_info.parent_id}),await this.freshRecordInfo()}async deleteOldFiles(){const e=this.mediaSubtitles.record_info.manifest,t=this.old_detail.manifest,{preview_file_name:i}=this;if(t?.parts[i]){if(t.baidu){const s=t.baidu.info[i]?.fs_id,a=e.baidu.info[i]?.fs_id;s&&s!==a&&await this.deleteBaiduHostFile({fs_id:s,host_id:this.mediaSubtitles.record_info.cloud_info.baidu.id})}if(t.ali){const s=t.ali.info[i]?.file_id,a=e.ali.info[i]?.file_id;s&&s!=a&&await this.deleteAliHostFile({file_id:t.ali.info[i]?.file_id,host_id:this.mediaSubtitles.record_info.cloud_info.ali.id})}}}async destroy(){const e=await this.repository.findOneBy({id:this.id});await(0,fs_extra_1.remove)(e.output_root),await this.repository.delete(this.id)}async freshRecordInfo(){const e=await this.repository.findOneBy({id:this.id}),{space_id:t,record_id:i}=e,s=await this.sdk.getRecordInfo({space_id:t,record_id:i,refresh:!0});await this.repository.update(this.id,{record_info:s}),this.mediaSubtitles=await this.repository.findOneBy({id:this.id})}async getBaiduAuthData(){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[types_1.HostType.BAIDU].id,refresh:!1})).access_token}}async getAliAuthData(){const e=await this.sdk.getAliHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[types_1.HostType.ALI].id,refresh:!1});return{drive_id:e.drive_id,access_token:e.access_token}}async ffmpeg(e,t){const{ffmpegPath:i}=this;return await new Promise(((s,a)=>{const r=(0,child_process_1.spawn)(i,e,Object.assign({stdio:"ignore"},t));r.on("close",(e=>{s(e)})),r.on("error",(e=>{a(e)}))}))}async initPreviewFileName(){const{record_info:e}=this.mediaSubtitles;if(!e?.manifest?.media)return;const{media:t}=e.manifest,{texts:i,parts:s}=t;if(i?.length)this.preview_file_name=i[0].file;else{let e=0;if(s?.length){const t=s.filter((e=>e.startsWith("preview_"))).map((e=>parseInt(e.replace("preview_","0"),10))).sort().reverse();t.length&&(e=t[0]+1)}this.preview_file_name=`preview_${e}`}}calculateData(){const{record_info:e,output_root:t}=this.mediaSubtitles,{vtt_infos:i,segments:s}=this,a=[],{length:r}=i;let o=0;s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:0,end:31,size:32}),o+=32;for(let e=0;e<r;e++){const t=i[e],{bin:r,source:d,...l}=t,n={...l,file:this.preview_file_name,uuid:(0,uuid_1.v4)(),start:o,end:o+t.size-1,source:null};s.push(r),o+=t.size,n.source={file:this.preview_file_name,start:o,end:o+t.source.size-1,size:t.source.size,ext:t.source.ext},s.push(t.source.bin),a.push(n),o+=t.source.size}const{media:d}=e.manifest;return(d.texts||[]).forEach((e=>{const i={...e,start:o,end:o+e.size-1,source:null};o+=e.size,s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:e.start,end:e.end,size:e.size}),e.source&&(i.source={...e.source,start:o,end:o+e.size-1},o+=e.source.size,s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:e.source.start,end:e.source.end,size:e.source.size})),a.push(i)})),a}async generateVtt(e){const t=(0,uuid_1.v4)(),{ext:i}=(0,path_1.parse)(e),{output_root:s}=this.mediaSubtitles,a=`${t}.vtt`,r=(0,path_1.resolve)(s,a),o=`${t}.zip`,d=(0,path_1.resolve)(s,o),l=`formatted_input${i}`,n=(0,path_1.resolve)(s,l),_=`${(0,uuid_1.v4)()}.zip`,u=(0,path_1.resolve)(s,_);if(await(0,utils_1.isUtf8File)(e))await(0,fs_extra_1.copy)(e,n);else{await(0,utils_1.saveFileAsUtf8)(e,n);if(!await(0,utils_1.isValidFile)(n))return}const h=["-i",n,"-c","webvtt","-y",a];await this.ffmpeg(h,{cwd:s});if(await(0,utils_1.isValidFile)(r)){const{label:t,lang:s}=await(0,utils_1.getFileLanguage)(r);await(0,utils_1.gzip)(e,u),await(0,utils_1.gzip)(r,d);const a=await(0,utils_3.getFileSize)(u),l=await(0,utils_3.getFileSize)(d);return{file:o,size:l,lang:s,name:t,title:t||(0,path_1.parse)(e).name,builtin:!1,bin:{file:d,start:0,end:l-1,size:l},source:{file:_,start:0,end:a-1,size:a,ext:i,bin:{file:u,start:0,end:a-1,size:a}}}}}async generateSafeFile(){const{record_info:e,output_root:t}=this.mediaSubtitles,{manifest:i}=e,s=(0,utils_2.getTheMd5)(i.meta.filesize);await(0,fs_extra_1.writeFile)((0,path_1.resolve)(t,this.old_bin_name),s)}}exports.Add=Add;

package/dist/common.d.ts

@@ -1,20 +0,0 @@
-import { Sdk } from '@soga/sdk';
-export declare class Common {
-    protected sdk_domain: string;
-    protected readonly sdk: Sdk;
-    constructor(sdk_domain: string);
-    protected deleteBaiduHostFile({ fs_id, host_id, }: {
-        fs_id: number;
-        host_id: number;
-    }): Promise<void>;
-    protected deleteAliHostFile({ host_id, file_id, }: {
-        host_id: number;
-        file_id: string;
-    }): Promise<void>;
-    protected getBaiduAuthData({ host_id }: {
-        host_id: number;
-    }): Promise<{
-        access_token: string;
-    }>;
-    private getSearchParams;
-}

package/dist/common.js

@@ -1 +0,0 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.Common=void 0;const sdk_1=require("@soga/sdk"),axios_1=__importDefault(require("axios"));class Common{sdk_domain;sdk;constructor(t){this.sdk_domain=t,this.sdk=new sdk_1.Sdk(this.sdk_domain)}async deleteBaiduHostFile({fs_id:t,host_id:e}){const{access_token:s}=await this.getBaiduAuthData({host_id:e}),a=`https://pan.baidu.com/rest/2.0/xpan/multimedia?method=filemetas&access_token=${s}&fsids=%5B${t}%5D`,i=await axios_1.default.get(a),{errno:o,list:d}=i.data;if(o)return;const r=d[0]?.path;if(r){const t=this.getSearchParams({async:"2",filelist:JSON.stringify([r])});await axios_1.default.post(`https://pan.baidu.com/rest/2.0/xpan/file?method=filemanager&access_token=${s}&opera=delete`,t)}}async deleteAliHostFile({host_id:t,file_id:e}){const{access_token:s,drive_id:a}=await this.sdk.getAliHostAuthData({host_id:t});await axios_1.default.post("https://openapi.alipan.com/adrive/v1.0/openFile/delete",{drive_id:a,file_id:e},{headers:{Authorization:`Bearer ${s}`}})}async getBaiduAuthData({host_id:t}){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:t,refresh:!1})).access_token}}getSearchParams(t){Object.keys(t).forEach((e=>{void 0===t[e]&&delete t[e]}));return new URLSearchParams(t).toString()}}exports.Common=Common;

package/dist/delete.d.ts

@@ -1,10 +0,0 @@
-import { DeleteSubtitleParams } from './types';
-import { Common } from './common';
-export declare class DeleteSubtitle extends Common {
-    protected readonly space_id: number;
-    protected readonly record_id: number;
-    protected readonly uuid: string;
-    constructor({ sdk_domain, space_id, record_id, uuid }: DeleteSubtitleParams);
-    start(): Promise<void>;
-    private getRecordInfo;
-}

package/dist/delete.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DeleteSubtitle=void 0;const common_1=require("./common");class DeleteSubtitle extends common_1.Common{space_id;record_id;uuid;constructor({sdk_domain:e,space_id:i,record_id:t,uuid:s}){super(e),this.space_id=i,this.record_id=t,this.uuid=s}async start(){const{space_id:e,record_id:i,uuid:t}=this,s=await this.getRecordInfo();if(!s.manifest.media?.texts?.length)return;const{manifest:d,parent_id:a}=s,o=d.media.texts[0].file;d.media.texts=s.manifest.media.texts.filter((e=>e.uuid!==t));const{length:r}=d.media.texts;let c=0,n="";if(!r){const{media:e}=d,{parts:i}=e;delete d.parts[o],d.media.parts=i.filter((e=>e!=o)),d.media.parts=d.media.parts.filter((e=>e!=o));let t=0;Object.values(d.parts).forEach((e=>{t+=e.size})),d.meta.host_size=t,d.baidu?.info[o]&&(c=d.baidu.info[o].fs_id,delete d.baidu.info[o]),d.ali?.info[o]&&(n=d.ali.info[o].file_id,delete d.ali.info[o])}await this.sdk.updateRecord({space_id:e,record_id:i,manifest:JSON.stringify(d),parent_id:a}),await this.getRecordInfo(),c&&await this.deleteBaiduHostFile({fs_id:c,host_id:d.baidu.host_id}),n&&await this.deleteAliHostFile({host_id:d.ali.host_id,file_id:n})}async getRecordInfo(e=!0){return await this.sdk.getRecordInfo({space_id:this.space_id,record_id:this.record_id,refresh:e})}}exports.DeleteSubtitle=DeleteSubtitle;

package/dist/types.d.ts

@@ -1,16 +0,0 @@
-import { DataSource } from 'typeorm';
-export interface CommonSubtitlesParams {
-    ffmpegPath: string;
-    sdk_domain: string;
-    dataSource: DataSource;
-    id: number;
-    debug?: boolean;
-}
-export interface AddSubtitlesParams extends CommonSubtitlesParams {
-}
-export interface DeleteSubtitleParams {
-    sdk_domain: string;
-    space_id: number;
-    record_id: number;
-    uuid: string;
-}

package/dist/types.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});

package/dist/utils.d.ts

@@ -1,6 +0,0 @@
-export declare function mergeFileFragments(fragments: {
-    file: string;
-    start: number;
-    end: number;
-}[], outputPath: string): Promise<void>;
-export declare function getTheMd5(text: string | number): string;

package/dist/utils.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.mergeFileFragments=mergeFileFragments,exports.getTheMd5=getTheMd5;const crypto_1=require("crypto"),fs_1=require("fs"),fs_extra_1=require("fs-extra");async function mergeFileFragments(e,t){let r=!1;const s=(0,fs_1.createWriteStream)(t);try{s.on("open",(()=>{r=!0})).on("error",(e=>{throw r&&s.close(),e}));for(const o of e){o.end,o.start;const e=(0,fs_1.createReadStream)(o.file,{start:o.start,end:o.end});await new Promise(((o,n)=>{e.pipe(s,{end:!1}),e.on("end",(()=>{e.close(),o(!0)})),e.on("error",(async o=>{e.destroy(),r&&s.close(),await(0,fs_extra_1.remove)(t),n(o)}))}))}s.end()}catch(e){throw s.destroy(),e}}function getTheMd5(e){const t=(0,crypto_1.createHash)("md5");return t.update(`${e}`),t.digest("hex")}