@soga/subtitles 0.3.0

package/README.md

@@ -0,0 +1 @@
+### @soga/baidu-ua

package/dist/add.d.ts

@@ -0,0 +1,93 @@
+import { AddSubtitlesParams } from './types';
+import { RecordDetail } from '@soga/types';
+import { SpawnOptions } from 'child_process';
+import { DataSource, Repository } from 'typeorm';
+import { MediaSubtitles } from '@soga/entities';
+import { Common } from './common';
+export declare class Add extends Common {
+    protected dataSource: DataSource;
+    protected repository: Repository<MediaSubtitles>;
+    protected readonly ffmpegPath: string;
+    protected readonly id: number;
+    protected preview_file_name: string;
+    protected old_bin_name: string;
+    protected old_detail: RecordDetail;
+    protected get old_exist(): boolean;
+    protected mediaSubtitles: MediaSubtitles;
+    protected texts: any[];
+    protected segments: {
+        file: string;
+        start: number;
+        end: number;
+        size: number;
+    }[];
+    private vtt_infos;
+    constructor({ dataSource, sdk_domain, ffmpegPath, id }: AddSubtitlesParams);
+    start(): Promise<{
+        state: boolean;
+        message: any;
+    }>;
+    init(): Promise<void>;
+    downloadFile(): Promise<void>;
+    generateVtts(): Promise<void>;
+    generateBin(): Promise<void>;
+    uploadFile(): Promise<void>;
+    updateRecordInfo(): Promise<void>;
+    deleteOldFiles(): Promise<void>;
+    destroy(): Promise<void>;
+    protected freshRecordInfo(): Promise<void>;
+    protected getBaiduAuthData(): Promise<{
+        access_token: string;
+    }>;
+    protected getAliAuthData(): Promise<{
+        drive_id: string;
+        access_token: string;
+    }>;
+    protected ffmpeg(args: string[], options?: SpawnOptions): Promise<unknown>;
+    protected initPreviewFileName(): Promise<void>;
+    private calculateData;
+    private generateVtt;
+    private generateSafeFile;
+}
+export type VideoSubtitleData = {
+    file: string;
+    size: number;
+    lang: string;
+    name: string;
+    title: string;
+    builtin: boolean;
+    source: null | {
+        file: string;
+        start: number;
+        end: number;
+        size: number;
+        ext: string;
+    };
+};
+export type VideoSubtitleInfo = {
+    file: string;
+    size: number;
+    lang: string;
+    name: string;
+    title: string;
+    builtin: boolean;
+    bin: {
+        file: string;
+        start: number;
+        end: number;
+        size: number;
+    };
+    source: null | {
+        file: string;
+        start: number;
+        end: number;
+        size: number;
+        ext: string;
+        bin: {
+            file: string;
+            start: number;
+            end: number;
+            size: number;
+        };
+    };
+};

package/dist/add.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Add=void 0;const utils_1=require("@soga/utils"),path_1=require("path"),uuid_1=require("uuid"),utils_2=require("./utils"),types_1=require("@soga/types"),child_process_1=require("child_process"),fs_extra_1=require("fs-extra"),fetcher_1=require("@soga/fetcher"),entities_1=require("@soga/entities"),part_uploader_1=require("@soga/part-uploader"),utils_3=require("@soga/utils"),common_1=require("./common");class Add extends common_1.Common{dataSource;repository;ffmpegPath;id;preview_file_name;old_bin_name="old.bin";old_detail;get old_exist(){return!!this.old_detail?.manifest?.media?.texts?.length}mediaSubtitles;texts=[];segments=[];vtt_infos=[];constructor({dataSource:e,sdk_domain:t,ffmpegPath:i,id:s}){super(t),this.id=s,this.dataSource=e,this.ffmpegPath=i,this.repository=e.getRepository(entities_1.MediaSubtitles)}async start(){try{return await this.init(),await this.downloadFile(),await this.generateVtts(),await this.generateBin(),await this.uploadFile(),await this.updateRecordInfo(),await this.deleteOldFiles(),await this.destroy(),{state:!0,message:"Subtitles added successfully."}}catch(e){return{state:!1,message:e.message}}}async init(){const e=await this.repository.findOneBy({id:this.id});this.old_detail=e.record_info,this.mediaSubtitles=await this.repository.findOneBy({id:this.id}),await this.initPreviewFileName(),await(0,fs_extra_1.ensureDir)(this.mediaSubtitles.output_root)}async downloadFile(){if(!this.old_exist)return;const{record_info:e,record_id:t,output_root:i}=this.mediaSubtitles;if(!e?.cloud_info)return;if(!e?.manifest?.media?.texts?.length)return;const{manifest:s}=e,a=e.cloud_info[types_1.HostType.BAIDU];e.cloud_info[types_1.HostType.ALI];if(a){const e=s.parts[this.preview_file_name],{access_token:r}=await this.getBaiduAuthData(),o=this.dataSource,d=o.getRepository(entities_1.FetchUrl),l=(await d.find({}),await(0,fetcher_1.fetchBaiduBuffer)({dataSource:o,record_id:t,start:0,end:e.size-1,total:e.size,access_token:r,host_id:a.id,host_vip_type:a.vip_type,fs_id:s.baidu.info[this.preview_file_name]?.fs_id,part_md5:e.md5,abort_controller:new AbortController})),n=(0,path_1.resolve)(i,this.old_bin_name);await(0,fs_extra_1.writeFile)(n,l)}}async generateVtts(){const{inputs:e}=this.mediaSubtitles,{length:t}=e;for(let i=0;i<t;i++){const t=e[i],s=await this.generateVtt(t);s&&this.vtt_infos.push(s)}}async generateBin(){this.old_exist||await this.generateSafeFile(),this.texts=this.calculateData();const e=(0,path_1.resolve)(this.mediaSubtitles.output_root,this.preview_file_name);await(0,utils_2.mergeFileFragments)(this.segments,e)}async uploadFile(){const{cloud_info:e}=this.mediaSubtitles.record_info;if(!e)return;const t=(0,path_1.resolve)(this.mediaSubtitles.output_root,this.preview_file_name),i=await(0,utils_3.getFileSize)(t),s=await(0,utils_3.calculateMd5)({file:t,start:0,end:i-1}),a={file_id:1,part_id:1,output_root:this.mediaSubtitles.output_root,sdk_domain:this.sdk_domain},r={file:this.preview_file_name,start:0,end:i-1,size:i,path:t,index:0,md5:s};if(e.baidu){const o=await(0,utils_3.calculateMd4)({file:t,start:0,end:i-1}),{fs_id:d}=await(0,part_uploader_1.uploadBaidu)({...a,host_id:e.baidu.id,cloud_folder_path:e.baidu.path,part:{...r,md4:o}});await this.repository.update(this.id,{baidu_upload_result:{md5:s,size:i,md4:o,fs_id:d}})}if(e.ali){const o=await(0,utils_3.calculateSha1)({file:t,start:0,end:i-1}),{file_id:d}=await(0,part_uploader_1.uploadAli)({...a,host_id:e.ali.id,cloud_folder_id:e.ali.file_id,part:{...r,sha1:o}});await this.repository.update(this.id,{ali_upload_result:{md5:s,size:i,sha1:o,file_id:d}})}this.mediaSubtitles=await this.repository.findOneBy({id:this.id})}async updateRecordInfo(){const e=this.preview_file_name,{space_id:t,record_id:i,record_info:s,baidu_upload_result:a,ali_upload_result:r,output_root:o}=this.mediaSubtitles,{manifest:d,manifest_md5:l}=s;if(!s)return;const n=(0,path_1.resolve)(o,this.preview_file_name),_=await(0,utils_3.getFileSize)(n),u=await(0,utils_3.calculateMd5)({file:n,start:0,end:_-1});this.old_exist?(d.parts[e].md5=u,d.parts[e].size=_):(d.parts[e]={md5:u,preview:!0,size:_,source:!1},d.media.parts?.length?d.media.parts.push(e):d.media.parts=[e]),a&&(d.baidu.info[e]={fs_id:a.fs_id,md4:a.md4}),r&&(d.ali.info[e]={file_id:r.file_id,sha1:r.sha1}),d.media.texts=this.texts;let h=0;Object.values(d.parts).forEach((e=>{h+=e.size})),d.meta.host_size=h;if((await this.sdk.getRecordInfo({space_id:t,record_id:i,refresh:!0})).manifest_md5!=l)throw new Error("File has been modified.");await this.sdk.updateRecord({space_id:this.mediaSubtitles.space_id,record_id:this.mediaSubtitles.record_id,manifest:JSON.stringify(d),parent_id:this.mediaSubtitles.record_info.parent_id}),await this.freshRecordInfo()}async deleteOldFiles(){const e=this.mediaSubtitles.record_info.manifest,t=this.old_detail.manifest,{preview_file_name:i}=this;if(t?.parts[i]){if(t.baidu){const s=t.baidu.info[i]?.fs_id,a=e.baidu.info[i]?.fs_id;s&&s!==a&&await this.deleteBaiduHostFile({fs_id:s,host_id:this.mediaSubtitles.record_info.cloud_info.baidu.id})}if(t.ali){const s=t.ali.info[i]?.file_id,a=e.ali.info[i]?.file_id;s&&s!=a&&await this.deleteAliHostFile({file_id:t.ali.info[i]?.file_id,host_id:this.mediaSubtitles.record_info.cloud_info.ali.id})}}}async destroy(){const e=await this.repository.findOneBy({id:this.id});await(0,fs_extra_1.remove)(e.output_root),await this.repository.delete(this.id)}async freshRecordInfo(){const e=await this.repository.findOneBy({id:this.id}),{space_id:t,record_id:i}=e,s=await this.sdk.getRecordInfo({space_id:t,record_id:i,refresh:!0});await this.repository.update(this.id,{record_info:s}),this.mediaSubtitles=await this.repository.findOneBy({id:this.id})}async getBaiduAuthData(){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[types_1.HostType.BAIDU].id,refresh:!1})).access_token}}async getAliAuthData(){const e=await this.sdk.getAliHostAuthData({host_id:this.mediaSubtitles.record_info.cloud_info[types_1.HostType.ALI].id,refresh:!1});return{drive_id:e.drive_id,access_token:e.access_token}}async ffmpeg(e,t){const{ffmpegPath:i}=this;return await new Promise(((s,a)=>{const r=(0,child_process_1.spawn)(i,e,Object.assign({stdio:"ignore"},t));r.on("close",(e=>{s(e)})),r.on("error",(e=>{a(e)}))}))}async initPreviewFileName(){const{record_info:e}=this.mediaSubtitles;if(!e?.manifest?.media)return;const{media:t}=e.manifest,{texts:i,parts:s}=t;if(i?.length)this.preview_file_name=i[0].file;else{let e=0;if(s?.length){const t=s.filter((e=>e.startsWith("preview_"))).map((e=>parseInt(e.replace("preview_","0"),10))).sort().reverse();t.length&&(e=t[0]+1)}this.preview_file_name=`preview_${e}`}}calculateData(){const{record_info:e,output_root:t}=this.mediaSubtitles,{vtt_infos:i,segments:s}=this,a=[],{length:r}=i;let o=0;s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:0,end:31,size:32}),o+=32;for(let e=0;e<r;e++){const t=i[e],{bin:r,source:d,...l}=t,n={...l,file:this.preview_file_name,uuid:(0,uuid_1.v4)(),start:o,end:o+t.size-1,source:null};s.push(r),o+=t.size,n.source={file:this.preview_file_name,start:o,end:o+t.source.size-1,size:t.source.size,ext:t.source.ext},s.push(t.source.bin),a.push(n),o+=t.source.size}const{media:d}=e.manifest;return(d.texts||[]).forEach((e=>{const i={...e,start:o,end:o+e.size-1,source:null};o+=e.size,s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:e.start,end:e.end,size:e.size}),e.source&&(i.source={...e.source,start:o,end:o+e.size-1},o+=e.source.size,s.push({file:(0,path_1.resolve)(t,this.old_bin_name),start:e.source.start,end:e.source.end,size:e.source.size})),a.push(i)})),a}async generateVtt(e){const t=(0,uuid_1.v4)(),{ext:i}=(0,path_1.parse)(e),{output_root:s}=this.mediaSubtitles,a=`${t}.vtt`,r=(0,path_1.resolve)(s,a),o=`${t}.zip`,d=(0,path_1.resolve)(s,o),l=`formatted_input${i}`,n=(0,path_1.resolve)(s,l),_=`${(0,uuid_1.v4)()}.zip`,u=(0,path_1.resolve)(s,_);if(await(0,utils_1.isUtf8File)(e))await(0,fs_extra_1.copy)(e,n);else{await(0,utils_1.saveFileAsUtf8)(e,n);if(!await(0,utils_1.isValidFile)(n))return}const h=["-i",n,"-c","webvtt","-y",a];await this.ffmpeg(h,{cwd:s});if(await(0,utils_1.isValidFile)(r)){const{label:t,lang:s}=await(0,utils_1.getFileLanguage)(r);await(0,utils_1.gzip)(e,u),await(0,utils_1.gzip)(r,d);const a=await(0,utils_3.getFileSize)(u),l=await(0,utils_3.getFileSize)(d);return{file:o,size:l,lang:s,name:t,title:t||(0,path_1.parse)(e).name,builtin:!1,bin:{file:d,start:0,end:l-1,size:l},source:{file:_,start:0,end:a-1,size:a,ext:i,bin:{file:u,start:0,end:a-1,size:a}}}}}async generateSafeFile(){const{record_info:e,output_root:t}=this.mediaSubtitles,{manifest:i}=e,s=(0,utils_2.getTheMd5)(i.meta.filesize);await(0,fs_extra_1.writeFile)((0,path_1.resolve)(t,this.old_bin_name),s)}}exports.Add=Add;

package/dist/common.d.ts

@@ -0,0 +1,20 @@
+import { Sdk } from '@soga/sdk';
+export declare class Common {
+    protected sdk_domain: string;
+    protected readonly sdk: Sdk;
+    constructor(sdk_domain: string);
+    protected deleteBaiduHostFile({ fs_id, host_id, }: {
+        fs_id: number;
+        host_id: number;
+    }): Promise<void>;
+    protected deleteAliHostFile({ host_id, file_id, }: {
+        host_id: number;
+        file_id: string;
+    }): Promise<void>;
+    protected getBaiduAuthData({ host_id }: {
+        host_id: number;
+    }): Promise<{
+        access_token: string;
+    }>;
+    private getSearchParams;
+}

package/dist/common.js

@@ -0,0 +1 @@
+"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.Common=void 0;const sdk_1=require("@soga/sdk"),axios_1=__importDefault(require("axios"));class Common{sdk_domain;sdk;constructor(t){this.sdk_domain=t,this.sdk=new sdk_1.Sdk(this.sdk_domain)}async deleteBaiduHostFile({fs_id:t,host_id:e}){const{access_token:s}=await this.getBaiduAuthData({host_id:e}),a=`https://pan.baidu.com/rest/2.0/xpan/multimedia?method=filemetas&access_token=${s}&fsids=%5B${t}%5D`,i=await axios_1.default.get(a),{errno:o,list:d}=i.data;if(o)return;const r=d[0]?.path;if(r){const t=this.getSearchParams({async:"2",filelist:JSON.stringify([r])});await axios_1.default.post(`https://pan.baidu.com/rest/2.0/xpan/file?method=filemanager&access_token=${s}&opera=delete`,t)}}async deleteAliHostFile({host_id:t,file_id:e}){const{access_token:s,drive_id:a}=await this.sdk.getAliHostAuthData({host_id:t});await axios_1.default.post("https://openapi.alipan.com/adrive/v1.0/openFile/delete",{drive_id:a,file_id:e},{headers:{Authorization:`Bearer ${s}`}})}async getBaiduAuthData({host_id:t}){return{access_token:(await this.sdk.getBaiduHostAuthData({host_id:t,refresh:!1})).access_token}}getSearchParams(t){Object.keys(t).forEach((e=>{void 0===t[e]&&delete t[e]}));return new URLSearchParams(t).toString()}}exports.Common=Common;

package/dist/delete.d.ts

@@ -0,0 +1,10 @@
+import { DeleteSubtitleParams } from './types';
+import { Common } from './common';
+export declare class DeleteSubtitle extends Common {
+    protected readonly space_id: number;
+    protected readonly record_id: number;
+    protected readonly uuid: string;
+    constructor({ sdk_domain, space_id, record_id, uuid }: DeleteSubtitleParams);
+    start(): Promise<void>;
+    private getRecordInfo;
+}

package/dist/delete.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DeleteSubtitle=void 0;const common_1=require("./common");class DeleteSubtitle extends common_1.Common{space_id;record_id;uuid;constructor({sdk_domain:e,space_id:i,record_id:t,uuid:s}){super(e),this.space_id=i,this.record_id=t,this.uuid=s}async start(){const{space_id:e,record_id:i,uuid:t}=this,s=await this.getRecordInfo();if(!s.manifest.media?.texts?.length)return;const{manifest:d,parent_id:a}=s,o=d.media.texts[0].file;d.media.texts=s.manifest.media.texts.filter((e=>e.uuid!==t));const{length:r}=d.media.texts;let c=0,n="";if(!r){const{media:e}=d,{parts:i}=e;delete d.parts[o],d.media.parts=i.filter((e=>e!=o)),d.media.parts=d.media.parts.filter((e=>e!=o));let t=0;Object.values(d.parts).forEach((e=>{t+=e.size})),d.meta.host_size=t,d.baidu?.info[o]&&(c=d.baidu.info[o].fs_id,delete d.baidu.info[o]),d.ali?.info[o]&&(n=d.ali.info[o].file_id,delete d.ali.info[o])}await this.sdk.updateRecord({space_id:e,record_id:i,manifest:JSON.stringify(d),parent_id:a}),await this.getRecordInfo(),c&&await this.deleteBaiduHostFile({fs_id:c,host_id:d.baidu.host_id}),n&&await this.deleteAliHostFile({host_id:d.ali.host_id,file_id:n})}async getRecordInfo(e=!0){return await this.sdk.getRecordInfo({space_id:this.space_id,record_id:this.record_id,refresh:e})}}exports.DeleteSubtitle=DeleteSubtitle;

package/dist/main.d.ts

@@ -0,0 +1,3 @@
+import { AddSubtitlesParams, DeleteSubtitleParams } from './types';
+export declare const addSubtitle: ({ dataSource, sdk_domain, ffmpegPath, id, }: AddSubtitlesParams) => Promise<void>;
+export declare const deleteSubtitle: (params: DeleteSubtitleParams) => Promise<void>;

package/dist/main.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.deleteSubtitle=exports.addSubtitle=void 0;const add_1=require("./add"),delete_1=require("./delete"),addSubtitle=async({dataSource:e,sdk_domain:t,ffmpegPath:d,id:a})=>{const i=new add_1.Add({dataSource:e,sdk_domain:t,ffmpegPath:d,id:a});await i.start()};exports.addSubtitle=addSubtitle;const deleteSubtitle=async e=>{const t=new delete_1.DeleteSubtitle(e);await t.start()};exports.deleteSubtitle=deleteSubtitle;

package/dist/types.d.ts

@@ -0,0 +1,16 @@
+import { DataSource } from 'typeorm';
+export interface CommonSubtitlesParams {
+    ffmpegPath: string;
+    sdk_domain: string;
+    dataSource: DataSource;
+    id: number;
+    debug?: boolean;
+}
+export interface AddSubtitlesParams extends CommonSubtitlesParams {
+}
+export interface DeleteSubtitleParams {
+    sdk_domain: string;
+    space_id: number;
+    record_id: number;
+    uuid: string;
+}

package/dist/types.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});

package/dist/utils.d.ts

@@ -0,0 +1,6 @@
+export declare function mergeFileFragments(fragments: {
+    file: string;
+    start: number;
+    end: number;
+}[], outputPath: string): Promise<void>;
+export declare function getTheMd5(text: string | number): string;

package/dist/utils.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.mergeFileFragments=mergeFileFragments,exports.getTheMd5=getTheMd5;const crypto_1=require("crypto"),fs_1=require("fs"),fs_extra_1=require("fs-extra");async function mergeFileFragments(e,t){let r=!1;const s=(0,fs_1.createWriteStream)(t);try{s.on("open",(()=>{r=!0})).on("error",(e=>{throw r&&s.close(),e}));for(const o of e){o.end,o.start;const e=(0,fs_1.createReadStream)(o.file,{start:o.start,end:o.end});await new Promise(((o,n)=>{e.pipe(s,{end:!1}),e.on("end",(()=>{e.close(),o(!0)})),e.on("error",(async o=>{e.destroy(),r&&s.close(),await(0,fs_extra_1.remove)(t),n(o)}))}))}s.end()}catch(e){throw s.destroy(),e}}function getTheMd5(e){const t=(0,crypto_1.createHash)("md5");return t.update(`${e}`),t.digest("hex")}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@soga/subtitles",
+  "version": "0.3.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "",
+  "main": "dist/main.js",
+  "types": "dist/main.d.ts",
+  "files": [
+    "dist"
+  ],
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@soga/entities": "^0.3.0",
+    "@soga/fetcher": "^0.3.0",
+    "@soga/part-uploader": "^0.3.0",
+    "@soga/sdk": "^0.3.0",
+    "@soga/types": "^0.3.0",
+    "@soga/utils": "^0.3.0",
+    "axios": "^1.9.0",
+    "typeorm": "^0.3.24",
+    "uuid": "^11.1.0"
+  },
+  "scripts": {
+    "build": "rimraf dist && tsc && ts-node ./scripts/minify",
+    "minify": "ts-node watch ./scripts/minify",
+    "demo": "ts-node  ./demo/demo.ts",
+    "test": "jest",
+    "dev": "tsc --watch",
+    "lint": "eslint . --ext .ts"
+  }
+}