@soga/task 1.3.8 → 1.3.12

package/dist/main.js

@@ -1 +1 @@
-var e,t=Object.create,i=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,l=(e,t,o,l)=>{if(t&&"object"==typeof t||"function"==typeof t)for(let d of s(t))r.call(e,d)||d===o||i(e,d,{get:()=>t[d],enumerable:!(l=a(t,d))||l.enumerable});return e},d={};((e,t)=>{for(var a in t)i(e,a,{get:t[a],enumerable:!0})})(d,{createDownloadTask:()=>K,createUploadTask:()=>R,runDownloadTask:()=>C,runUploadTask:()=>j}),module.exports=(e=d,l(i({},"__esModule",{value:!0}),e));var n=require("@soga/types"),c=require("path"),p=require("@soga/fileutils"),f=require("@soga/sdk"),_=require("@soga/entities"),h=require("path"),u=((e,a,s)=>(s=null!=e?t(o(e)):{},l(!a&&e&&e.__esModule?s:i(s,"default",{value:e,enumerable:!0}),e)))(require("xregexp")),y=(e,t)=>{const i=t.filter(t=>e.parent_path.startsWith(t.parent_path));if(!i.length)return null;const a=i.map(t=>{const i=(0,h.relative)(t.parent_path,e.parent_path).split(h.sep).filter(e=>!!e).length,a=t.keywords.filter(t=>e.filename.includes(t)).length/t.keywords.length;return{...t,percent:1e3*a/(10*i+1)}}).sort((e,t)=>t.percent-e.percent)[0];return a.percent<1?null:a},m=(e,t)=>{const i=[],a=[...e];let s=[...t];for(const e of s){const t=y(e,a);if(t){const{filepath:i}=t,s=a.find(e=>e.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(e.filepath))}else i.push(e)}return[a,i]},w=e=>{const t=e.split((0,u.default)("[^\\p{L}0-9]","g")).filter(e=>!!e);return[...new Set(t)]},g=require("@soga/error"),R=async e=>{const t=e.dataSource.getRepository(_.UploadFile);let i=[];try{const a=(0,f.getSdk)(e.sdk_domain),s=await a.getUserInfo({refresh:!1}),{id:o}=s,{cloud_info:r,album:l}=await a.getRecordInfo({space_id:e.space_id,record_id:e.record_id,refresh:!0}),d={file_keeps:l.config.file_keeps},_=d.file_keeps==n.RecordFileKeep.SOURCE||d.file_keeps==n.RecordFileKeep.BOTH,h=d.file_keeps==n.RecordFileKeep.PREVIEW||d.file_keeps==n.RecordFileKeep.BOTH,{name:u}=await a.getSpaceInfo({space_id:e.space_id,refresh:!0}),y=l.id;let R=null,b=null;r[n.HostType.ALI]&&(R=r[n.HostType.ALI].id),r[n.HostType.BAIDU]&&(b=r[n.HostType.BAIDU].id);const{paths:O}=e,k=[],F=[],T=[],D=[],{length:E}=O;for(let e=0;e<E;e++){const t=O[e].replace(/[/\\]$/,"");if(!await(0,p.isValidPath)(t))throw(0,g.buildDError)(new Error,{message:`Invalid file path: ${t}`,detail:`Invalid file path: ${t}`});{const e=await(0,p.getFileinfo)(t),{record_type:i,record_ftype:a}=e,{dir:s,base:o,name:r}=(0,c.parse)(t),l={type:i,ftype:a,title:r,filename:o,basepath:(0,c.join)(s,r),filepath:t,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,c.dirname)(t),file_type:"file"};if(i==n.RecordType.FOLDER)F.push(l);else{const e=i==n.RecordType.VIDEO||i==n.RecordType.AUDIO,a=(0,p.isSubtitle)(t);if(e){const e=w(l.title);D.push({...l,file_type:"media",keywords:e})}else a?T.push({...l,file_type:"subtitle"}):k.push(l)}}}const[q,v]=m(D,T),x=[..._?T:v,...h?q:D,...k,...F].map(t=>{const i={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:e.is_attachment,filepath:t.filepath,space_id:e.space_id,space_name:u,aid:y,ali_host_id:R,baidu_host_id:b,config:d,is_paused:!1,is_ready:t.type!=n.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:e.record_id,uid:o,total_count:t.type==n.RecordType.FOLDER?void 0:1};return"media"==t.file_type&&(i.external_texts=t.external_texts),i});if(!x.length)return;const B=x.filter(e=>e.type!=n.RecordType.FOLDER);if(B.length){const{length:e}=B,a=100;for(let s=0;s<e;s+=a){const e=x.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}const L=x.filter(e=>e.type==n.RecordType.FOLDER);if(L.length){const{length:e}=L,a=100;for(let s=0;s<e;s+=a){const e=L.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}}catch(a){try{i.length&&await t.delete(i)}catch(e){}throw(0,g.buildDError)(a,{message:"create upload task failed",detail:`create upload task failed: [${e.paths.join(", ")}]`})}},b=require("@soga/entities"),O=require("@soga/entities"),k=require("@soga/types"),F=require("fs-extra"),T=require("glob"),D=require("os"),E=require("path"),q=require("typeorm"),v=require("@soga/fileutils"),x=require("@soga/lowdb"),B=require("crypto"),L=require("@soga/error"),S=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(e){const t=(0,B.createHash)("md5");return t.update(`${e}`),t.digest("hex")}constructor({dataSource:e,file_id:t}){this.dataSource=e,this.fileRepository=e.getRepository(O.UploadFile),this.file_id=t}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(e){throw(0,L.buildDError)(e,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await this.fileRepository.countBy({root_id:this.file_id,type:(0,q.Not)(k.RecordType.FOLDER)});e.is_ready=!0,e.total_count=t,await this.fileRepository.save(e)}async insertFiles(){const{files_inserted:e,files:t,medias:i,subtitles:a}=this.db.data;if(e)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.SOURCE||o.file_keeps==k.RecordFileKeep.BOTH),l=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.PREVIEW||o.file_keeps==k.RecordFileKeep.BOTH),[d,n]=m(i,a),c=[...r?a:n,...t,...l?d:i],{length:p}=c;if(!p)return;const f=[];for(let e=0;e<p;e++){const t=c[e],{filepath:i}=t;if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=(0,E.dirname)(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,d={is_attachment:s.is_attachment,input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},filepath:t.filepath,space_id:s.space_id,space_name:s.space_name,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:t.type!=k.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==t.file_type&&(d.external_texts=t.external_texts),f.push(d)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const e=_.manager.getRepository(O.UploadFile),{length:t}=f,i=50;for(let a=0;a<t;a+=i){const t=f.slice(a,a+i);await e.createQueryBuilder().insert().into(O.UploadFile).values(t).execute()}await _.commitTransaction()}catch(e){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:e,folders_inserted:t}=this.db.data;if(t)return;const{length:i}=e,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let t=0;t<i;t++){const i=(0,E.resolve)(a.filepath,e[t]);if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=(0,E.resolve)(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,d={is_attachment:a.is_attachment,input:{filename:(0,E.basename)(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:k.RecordType.FOLDER,ftype:k.RecordFtype.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(d))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await(0,T.glob)("**/*",{nodir:!0,cwd:e.filepath});let i=[];t.map(e=>{let t=(0,E.dirname)(e);const a=[];for(;"."!=t&&!i.includes(t);)a.unshift(t),t=(0,E.dirname)(t);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=t;for(let i=0;i<r;i++){const r=(0,E.resolve)(e.filepath,t[i]);if(await(0,v.isValidPath)(r)){const e=await(0,v.getFileinfo)(r),{record_type:t,record_ftype:i}=e,{dir:l,base:d,name:n}=(0,E.parse)(r),c={type:t,ftype:i,title:n,filename:d,basepath:(0,E.join)(l,n),filepath:r,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,E.dirname)(r),file_type:"file"},p=t==k.RecordType.VIDEO||t==k.RecordType.AUDIO,f=(0,v.isSubtitle)(r);if(p){const e=w(c.title);o.push({...c,file_type:"media",keywords:e})}else f?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await(0,F.remove)(this.cache_file)}async init(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const{id:t,input:i}=e,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=(0,E.resolve)((0,D.homedir)(),".dpan","desktop","json_files",`${t}_${a}.json`);this.db=await(0,x.getDb)(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}},I=require("@soga/error"),U=new Map,j=async e=>{try{const t=U.get(e.uid);if(t)return t;U.set(e.uid,!0);const i=e.dataSource.getRepository(b.UploadFile),a=async()=>{const t=await i.findOneBy({uid:e.uid,root_id:0,is_ready:!1});if(!t)return;const s=new S({dataSource:e.dataSource,file_id:t.id});await s.start(),await a()};await a(),U.delete(e.uid)}catch(e){throw(0,I.buildDError)(e,{message:"run upload task failed",detail:"run upload task failed"})}},P=require("@soga/sdk"),z=require("@soga/types"),$=require("@soga/entities"),H=require("@soga/error"),K=async e=>{const{dataSource:t,sdk_domain:i,record_ids:a,space_id:s}=e,o=(0,P.getSdk)(i),{name:r}=await o.getSpaceInfo({space_id:e.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:d}=l,n=t.getRepository($.DownloadFile);let c=[];try{const{length:e}=a,t=[];for(let i=0;i<e;i++){const e=a[i];i>0&&await new Promise(e=>setTimeout(e,500));const l=await o.getRecordInfo({space_id:s,record_id:e,refresh:!0});if(!l?.manifest)continue;const n={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:d,detail:l,file_total:l.type==z.RecordType.FOLDER?void 0:1,is_ready:l.type!=z.RecordType.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};t.push(n)}const i=t.filter(e=>e.type!=z.RecordType.FOLDER);if(i.length){const e=i.length;for(let t=0;t<e;t++){const e=i[t],{info:a,...s}=e,o=(await n.save(n.create({...s})))[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=a;if(e.has_attachment){const e=r.manifest.attachments;for(const t of e){const{uuid:e,type:i,ftype:a,title:o}=t,r={...s,attachment_uuid:e,title:o,type:i,ftype:a},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}const l=t.filter(e=>e.type==z.RecordType.FOLDER);if(l.length){const e=l.length;for(let t=0;t<e;t++){const e=l[t],{info:i,...a}=e,s=await n.save(n.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=i,d=r.manifest?.attachments;for(const e of d){const{uuid:t,type:i,ftype:s,title:o}=e,r={...a,attachment_uuid:t,has_attachment:!1,title:o,type:i,ftype:s},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}catch(e){try{c.length&&await n.delete(c)}catch(e){}throw(0,H.buildDError)(e,{message:"create download task failed",detail:"create download task failed"})}},V=require("@soga/error"),A=require("@soga/entities"),M=require("@soga/types"),N=require("@soga/sdk"),W=require("typeorm"),C=async e=>{const{dataSource:t,sdk_domain:i,space_id:a,uid:s}=e,o=(0,N.getSdk)(i),r=t.getRepository(A.DownloadFile);try{const e=await r.findBy({uid:s,root_id:0,is_ready:!1}),t=async e=>{let i=1,s=1e3,l=[];const d=0==e.root_id?e.id:e.root_id;for(;s>=100;){const t=await o.getRecordList({space_id:a,parent_id:e.detail.id,page:i,pagesize:100});t?(s=t.length,l=l.concat(t),i++):s=0}if(l.length){const i=l.map(t=>({title:t.name,cloud_id:t.id,baidu_host_id:e.baidu_host_id,ali_host_id:e.ali_host_id,pid:e.id,type:t.type,ftype:t.ftype,routes:[...e.routes,t.name],root_id:d,space_id:t.space_id,space_name:e.space_name,uid:e.uid,detail:t,file_total:t.type==M.RecordType.FOLDER?void 0:1})),a=i.filter(e=>e.type!=M.RecordType.FOLDER);for(const e of a){await r.findOneBy({root_id:d,cloud_id:e.cloud_id})||await r.insert(e)}const s=i.filter(e=>e.type==M.RecordType.FOLDER);for(const e of s){const i=await r.findOneBy({root_id:d,cloud_id:e.cloud_id});if(i)await t(i);else{const i=await r.save(r.create(e));await t(i)}}}};for(const i of e){await t(i);const e=await r.countBy({root_id:i.id,type:(0,W.Not)(M.RecordType.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:e})}}catch(e){throw(0,V.buildDError)(e,{message:"run download task failed",detail:"run download task failed"})}};
+var e,t=Object.create,i=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,l=(e,t,o,l)=>{if(t&&"object"==typeof t||"function"==typeof t)for(let d of s(t))r.call(e,d)||d===o||i(e,d,{get:()=>t[d],enumerable:!(l=a(t,d))||l.enumerable});return e},d={};((e,t)=>{for(var a in t)i(e,a,{get:t[a],enumerable:!0})})(d,{createDownloadTask:()=>K,createUploadTask:()=>R,runDownloadTask:()=>C,runUploadTask:()=>j}),module.exports=(e=d,l(i({},"__esModule",{value:!0}),e));var n=require("@soga/types"),c=require("path"),p=require("@soga/fileutils"),f=require("@soga/sdk"),_=require("@soga/entities"),h=require("path"),u=((e,a,s)=>(s=null!=e?t(o(e)):{},l(!a&&e&&e.__esModule?s:i(s,"default",{value:e,enumerable:!0}),e)))(require("xregexp")),y=(e,t)=>{const i=t.filter(t=>e.parent_path.startsWith(t.parent_path));if(!i.length)return null;const a=i.map(t=>{const i=(0,h.relative)(t.parent_path,e.parent_path).split(h.sep).filter(e=>!!e).length,a=t.keywords.filter(t=>e.filename.includes(t)).length/t.keywords.length;return{...t,percent:1e3*a/(10*i+1)}}).sort((e,t)=>t.percent-e.percent)[0];return a.percent<1?null:a},m=(e,t)=>{const i=[],a=[...e];let s=[...t];for(const e of s){const t=y(e,a);if(t){const{filepath:i}=t,s=a.find(e=>e.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(e.filepath))}else i.push(e)}return[a,i]},w=e=>{const t=e.split((0,u.default)("[^\\p{L}0-9]","g")).filter(e=>!!e);return[...new Set(t)]},g=require("@soga/error"),R=async e=>{const t=e.dataSource.getRepository(_.UploadFile);let i=[];try{const a=(0,f.getSdk)(e.sdk_domain),s=await a.getUserInfo({refresh:!1}),{id:o,private_space_id:r}=s,{cloud_info:l,album:d}=await a.getRecordInfo({space_id:e.space_id,record_id:e.record_id,refresh:!0}),_={file_keeps:d.config.file_keeps},h=_.file_keeps==n.RecordFileKeep.SOURCE||_.file_keeps==n.RecordFileKeep.BOTH,u=_.file_keeps==n.RecordFileKeep.PREVIEW||_.file_keeps==n.RecordFileKeep.BOTH,{name:y}=await a.getSpaceInfo({space_id:e.space_id,refresh:!0}),R=d.id;let b=null,O=null;l[n.HostType.ALI]&&(b=l[n.HostType.ALI].id),l[n.HostType.BAIDU]&&(O=l[n.HostType.BAIDU].id);const{paths:k}=e,F=[],T=[],D=[],E=[],{length:q}=k;for(let e=0;e<q;e++){const t=k[e].replace(/[/\\]$/,"");if(!await(0,p.isValidPath)(t))throw(0,g.buildDError)(new Error,{message:`Invalid file path: ${t}`,detail:`Invalid file path: ${t}`});{const e=await(0,p.getFileinfo)(t),{record_type:i,record_ftype:a}=e,{dir:s,base:o,name:r}=(0,c.parse)(t),l={type:i,ftype:a,title:r,filename:o,basepath:(0,c.join)(s,r),filepath:t,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,c.dirname)(t),file_type:"file"};if(i==n.RecordType.FOLDER)T.push(l);else{const e=i==n.RecordType.VIDEO||i==n.RecordType.AUDIO,a=(0,p.isSubtitle)(t);if(e){const e=w(l.title);E.push({...l,file_type:"media",keywords:e})}else a?D.push({...l,file_type:"subtitle"}):F.push(l)}}}const[v,x]=m(E,D),B=[...h?D:x,...u?v:E,...F,...T].map(t=>{const i={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:e.is_attachment,filepath:t.filepath,space_id:e.space_id,space_name:y,is_shop_space:r!=e.space_id,aid:R,ali_host_id:b,baidu_host_id:O,config:_,is_paused:!1,is_ready:t.type!=n.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:e.record_id,uid:o,total_count:t.type==n.RecordType.FOLDER?void 0:1};return"media"==t.file_type&&(i.external_texts=t.external_texts),i});if(!B.length)return;const L=B.filter(e=>e.type!=n.RecordType.FOLDER);if(L.length){const{length:e}=L,a=100;for(let s=0;s<e;s+=a){const e=B.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}const S=B.filter(e=>e.type==n.RecordType.FOLDER);if(S.length){const{length:e}=S,a=100;for(let s=0;s<e;s+=a){const e=S.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}}catch(a){try{i.length&&await t.delete(i)}catch(e){}throw(0,g.buildDError)(a,{message:"create upload task failed",detail:`create upload task failed: [${e.paths.join(", ")}]`})}},b=require("@soga/entities"),O=require("@soga/entities"),k=require("@soga/types"),F=require("fs-extra"),T=require("glob"),D=require("os"),E=require("path"),q=require("typeorm"),v=require("@soga/fileutils"),x=require("@soga/lowdb"),B=require("crypto"),L=require("@soga/error"),S=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(e){const t=(0,B.createHash)("md5");return t.update(`${e}`),t.digest("hex")}constructor({dataSource:e,file_id:t}){this.dataSource=e,this.fileRepository=e.getRepository(O.UploadFile),this.file_id=t}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(e){throw(0,L.buildDError)(e,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await this.fileRepository.countBy({root_id:this.file_id,type:(0,q.Not)(k.RecordType.FOLDER)});e.is_ready=!0,e.total_count=t,await this.fileRepository.save(e)}async insertFiles(){const{files_inserted:e,files:t,medias:i,subtitles:a}=this.db.data;if(e)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.SOURCE||o.file_keeps==k.RecordFileKeep.BOTH),l=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.PREVIEW||o.file_keeps==k.RecordFileKeep.BOTH),[d,n]=m(i,a),c=[...r?a:n,...t,...l?d:i],{length:p}=c;if(!p)return;const f=[];for(let e=0;e<p;e++){const t=c[e],{filepath:i}=t;if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=(0,E.dirname)(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,d={is_attachment:s.is_attachment,input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},filepath:t.filepath,space_id:s.space_id,space_name:s.space_name,is_shop_space:s.is_shop_space,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:t.type!=k.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==t.file_type&&(d.external_texts=t.external_texts),f.push(d)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const e=_.manager.getRepository(O.UploadFile),{length:t}=f,i=50;for(let a=0;a<t;a+=i){const t=f.slice(a,a+i);await e.createQueryBuilder().insert().into(O.UploadFile).values(t).execute()}await _.commitTransaction()}catch(e){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:e,folders_inserted:t}=this.db.data;if(t)return;const{length:i}=e,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let t=0;t<i;t++){const i=(0,E.resolve)(a.filepath,e[t]);if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=(0,E.resolve)(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,d={is_attachment:a.is_attachment,input:{filename:(0,E.basename)(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:k.RecordType.FOLDER,ftype:k.RecordFtype.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(d))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await(0,T.glob)("**/*",{nodir:!0,cwd:e.filepath});let i=[];t.map(e=>{let t=(0,E.dirname)(e);const a=[];for(;"."!=t&&!i.includes(t);)a.unshift(t),t=(0,E.dirname)(t);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=t;for(let i=0;i<r;i++){const r=(0,E.resolve)(e.filepath,t[i]);if(await(0,v.isValidPath)(r)){const e=await(0,v.getFileinfo)(r),{record_type:t,record_ftype:i}=e,{dir:l,base:d,name:n}=(0,E.parse)(r),c={type:t,ftype:i,title:n,filename:d,basepath:(0,E.join)(l,n),filepath:r,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,E.dirname)(r),file_type:"file"},p=t==k.RecordType.VIDEO||t==k.RecordType.AUDIO,f=(0,v.isSubtitle)(r);if(p){const e=w(c.title);o.push({...c,file_type:"media",keywords:e})}else f?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await(0,F.remove)(this.cache_file)}async init(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const{id:t,input:i}=e,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=(0,E.resolve)((0,D.homedir)(),".dpan","desktop","json_files",`${t}_${a}.json`);this.db=await(0,x.getDb)(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}},I=require("@soga/error"),U=new Map,j=async e=>{try{const t=U.get(e.uid);if(t)return t;U.set(e.uid,!0);const i=e.dataSource.getRepository(b.UploadFile),a=async()=>{const t=await i.findOneBy({uid:e.uid,root_id:0,is_ready:!1});if(!t)return;const s=new S({dataSource:e.dataSource,file_id:t.id});await s.start(),await a()};await a(),U.delete(e.uid)}catch(e){throw(0,I.buildDError)(e,{message:"run upload task failed",detail:"run upload task failed"})}},P=require("@soga/sdk"),z=require("@soga/types"),$=require("@soga/entities"),H=require("@soga/error"),K=async e=>{const{dataSource:t,sdk_domain:i,record_ids:a,space_id:s}=e,o=(0,P.getSdk)(i),{name:r}=await o.getSpaceInfo({space_id:e.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:d}=l,n=t.getRepository($.DownloadFile);let c=[];try{const{length:e}=a,t=[];for(let i=0;i<e;i++){const e=a[i];i>0&&await new Promise(e=>setTimeout(e,500));const l=await o.getRecordInfo({space_id:s,record_id:e,refresh:!0}),n={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:d,detail:l,file_total:l.type==z.RecordType.FOLDER?void 0:1,is_ready:l.type!=z.RecordType.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};t.push(n)}const i=t.filter(e=>e.type!=z.RecordType.FOLDER);if(i.length){const e=i.length;for(let t=0;t<e;t++){const e=i[t],{info:a,...s}=e,o=(await n.save(n.create({...s})))[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=a;if(e.has_attachment){const e=r.manifest.attachments;for(const t of e){const{uuid:e,type:i,ftype:a,title:o}=t,r={...s,attachment_uuid:e,title:o,type:i,ftype:a},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}const l=t.filter(e=>e.type==z.RecordType.FOLDER);if(l.length){const e=l.length;for(let t=0;t<e;t++){const e=l[t],{info:i,...a}=e,s=await n.save(n.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=i,d=r.manifest?.attachments;for(const e of d){const{uuid:t,type:i,ftype:s,title:o}=e,r={...a,attachment_uuid:t,has_attachment:!1,title:o,type:i,ftype:s},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}catch(e){try{c.length&&await n.delete(c)}catch(e){}throw(0,H.buildDError)(e,{message:"create download task failed",detail:"create download task failed"})}},V=require("@soga/error"),A=require("@soga/entities"),M=require("@soga/types"),N=require("@soga/sdk"),W=require("typeorm"),C=async e=>{const{dataSource:t,sdk_domain:i,space_id:a,uid:s}=e,o=(0,N.getSdk)(i),r=t.getRepository(A.DownloadFile);try{const e=await r.findBy({uid:s,root_id:0,is_ready:!1}),t=async e=>{let i=1,s=1e3,l=[];const d=0==e.root_id?e.id:e.root_id;for(;s>=100;){const t=await o.getRecordList({space_id:a,parent_id:e.detail.id,page:i,pagesize:100});t?(s=t.length,l=l.concat(t),i++):s=0}if(l.length){const i=l.map(t=>({title:t.name,cloud_id:t.id,baidu_host_id:e.baidu_host_id,ali_host_id:e.ali_host_id,pid:e.id,type:t.type,ftype:t.ftype,routes:[...e.routes,t.name],root_id:d,space_id:t.space_id,space_name:e.space_name,uid:e.uid,detail:t,file_total:t.type==M.RecordType.FOLDER?void 0:1})),a=i.filter(e=>e.type!=M.RecordType.FOLDER);for(const e of a){await r.findOneBy({root_id:d,cloud_id:e.cloud_id})||await r.insert(e)}const s=i.filter(e=>e.type==M.RecordType.FOLDER);for(const e of s){const i=await r.findOneBy({root_id:d,cloud_id:e.cloud_id});if(i)await t(i);else{const i=await r.save(r.create(e));await t(i)}}}};for(const i of e){await t(i);const e=await r.countBy({root_id:i.id,type:(0,W.Not)(M.RecordType.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:e})}}catch(e){throw(0,V.buildDError)(e,{message:"run download task failed",detail:"run download task failed"})}};

package/dist/main.mjs

@@ -1 +1 @@
-import{HostType as t,RecordFileKeep as e,RecordType as i}from"@soga/types";import{dirname as a,join as s,parse as o}from"path";import{getFileinfo as r,isSubtitle as l,isValidPath as n}from"@soga/fileutils";import{getSdk as d}from"@soga/sdk";import{UploadFile as c}from"@soga/entities";import{relative as f,sep as p}from"path";import _ from"xregexp";var h=(t,e)=>{const i=e.filter(e=>t.parent_path.startsWith(e.parent_path));if(!i.length)return null;const a=i.map(e=>{const i=f(e.parent_path,t.parent_path).split(p).filter(t=>!!t).length,a=e.keywords.filter(e=>t.filename.includes(e)).length/e.keywords.length;return{...e,percent:1e3*a/(10*i+1)}}).sort((t,e)=>e.percent-t.percent)[0];return a.percent<1?null:a},m=(t,e)=>{const i=[],a=[...t];let s=[...e];for(const t of s){const e=h(t,a);if(e){const{filepath:i}=e,s=a.find(t=>t.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(t.filepath))}else i.push(t)}return[a,i]},u=t=>{const e=t.split(_("[^\\p{L}0-9]","g")).filter(t=>!!t);return[...new Set(e)]};import{buildDError as y}from"@soga/error";var w=async f=>{const p=f.dataSource.getRepository(c);let _=[];try{const c=d(f.sdk_domain),h=await c.getUserInfo({refresh:!1}),{id:w}=h,{cloud_info:g,album:b}=await c.getRecordInfo({space_id:f.space_id,record_id:f.record_id,refresh:!0}),R={file_keeps:b.config.file_keeps},k=R.file_keeps==e.SOURCE||R.file_keeps==e.BOTH,O=R.file_keeps==e.PREVIEW||R.file_keeps==e.BOTH,{name:E}=await c.getSpaceInfo({space_id:f.space_id,refresh:!0}),D=b.id;let F=null,x=null;g[t.ALI]&&(F=g[t.ALI].id),g[t.BAIDU]&&(x=g[t.BAIDU].id);const{paths:B}=f,L=[],v=[],I=[],S=[],{length:z}=B;for(let t=0;t<z;t++){const e=B[t].replace(/[/\\]$/,"");if(!await n(e))throw y(new Error,{message:`Invalid file path: ${e}`,detail:`Invalid file path: ${e}`});{const t=await r(e),{record_type:n,record_ftype:d}=t,{dir:c,base:f,name:p}=o(e),_={type:n,ftype:d,title:p,filename:f,basepath:s(c,p),filepath:e,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:a(e),file_type:"file"};if(n==i.FOLDER)v.push(_);else{const t=n==i.VIDEO||n==i.AUDIO,a=l(e);if(t){const t=u(_.title);S.push({..._,file_type:"media",keywords:t})}else a?I.push({..._,file_type:"subtitle"}):L.push(_)}}}const[T,U]=m(S,I),$=[...k?I:U,...O?T:S,...L,...v].map(t=>{const e={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:f.is_attachment,filepath:t.filepath,space_id:f.space_id,space_name:E,aid:D,ali_host_id:F,baidu_host_id:x,config:R,is_paused:!1,is_ready:t.type!=i.FOLDER,type:t.type,ftype:t.ftype,task_record_id:f.record_id,uid:w,total_count:t.type==i.FOLDER?void 0:1};return"media"==t.file_type&&(e.external_texts=t.external_texts),e});if(!$.length)return;const A=$.filter(t=>t.type!=i.FOLDER);if(A.length){const{length:t}=A,e=100;for(let i=0;i<t;i+=e){const t=$.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}const V=$.filter(t=>t.type==i.FOLDER);if(V.length){const{length:t}=V,e=100;for(let i=0;i<t;i+=e){const t=V.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}}catch(t){try{_.length&&await p.delete(_)}catch(t){}throw y(t,{message:"create upload task failed",detail:`create upload task failed: [${f.paths.join(", ")}]`})}};import{UploadFile as g}from"@soga/entities";import{UploadFile as b}from"@soga/entities";import{RecordFileKeep as R,RecordFtype as k,RecordType as O}from"@soga/types";import{remove as E}from"fs-extra";import{glob as D}from"glob";import{homedir as F}from"os";import{resolve as x,dirname as B,parse as L,join as v,basename as I}from"path";import{Not as S}from"typeorm";import{getFileinfo as z,isSubtitle as T,isValidPath as U}from"@soga/fileutils";import{getDb as $}from"@soga/lowdb";import{createHash as A}from"crypto";import{buildDError as V}from"@soga/error";var j=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(t){const e=A("md5");return e.update(`${t}`),e.digest("hex")}constructor({dataSource:t,file_id:e}){this.dataSource=t,this.fileRepository=t.getRepository(b),this.file_id=e}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(t){throw V(t,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await this.fileRepository.countBy({root_id:this.file_id,type:S(O.FOLDER)});t.is_ready=!0,t.total_count=e,await this.fileRepository.save(t)}async insertFiles(){const{files_inserted:t,files:e,medias:i,subtitles:a}=this.db.data;if(t)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==R.SOURCE||o.file_keeps==R.BOTH),l=!!s.is_attachment||(o.file_keeps==R.PREVIEW||o.file_keeps==R.BOTH),[n,d]=m(i,a),c=[...r?a:d,...e,...l?n:i],{length:f}=c;if(!f)return;const p=[];for(let t=0;t<f;t++){const e=c[t],{filepath:i}=e;if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=B(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,n={is_attachment:s.is_attachment,input:{filename:e.filename,filepath:e.filepath,filesize:e.filesize,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime},filepath:e.filepath,space_id:s.space_id,space_name:s.space_name,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:e.type!=O.FOLDER,type:e.type,ftype:e.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==e.file_type&&(n.external_texts=e.external_texts),p.push(n)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const t=_.manager.getRepository(b),{length:e}=p,i=50;for(let a=0;a<e;a+=i){const e=p.slice(a,a+i);await t.createQueryBuilder().insert().into(b).values(e).execute()}await _.commitTransaction()}catch(t){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:t,folders_inserted:e}=this.db.data;if(e)return;const{length:i}=t,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let e=0;e<i;e++){const i=x(a.filepath,t[e]);if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=x(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,n={is_attachment:a.is_attachment,input:{filename:I(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:O.FOLDER,ftype:k.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(n))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await D("**/*",{nodir:!0,cwd:t.filepath});let i=[];e.map(t=>{let e=B(t);const a=[];for(;"."!=e&&!i.includes(e);)a.unshift(e),e=B(e);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=e;for(let i=0;i<r;i++){const r=x(t.filepath,e[i]);if(await U(r)){const t=await z(r),{record_type:e,record_ftype:i}=t,{dir:l,base:n,name:d}=L(r),c={type:e,ftype:i,title:d,filename:n,basepath:v(l,d),filepath:r,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:B(r),file_type:"file"},f=e==O.VIDEO||e==O.AUDIO,p=T(r);if(f){const t=u(c.title);o.push({...c,file_type:"media",keywords:t})}else p?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await E(this.cache_file)}async init(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const{id:e,input:i}=t,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=x(F(),".dpan","desktop","json_files",`${e}_${a}.json`);this.db=await $(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}};import{buildDError as H}from"@soga/error";var P=new Map,M=async t=>{try{const e=P.get(t.uid);if(e)return e;P.set(t.uid,!0);const i=t.dataSource.getRepository(g),a=async()=>{const e=await i.findOneBy({uid:t.uid,root_id:0,is_ready:!1});if(!e)return;const s=new j({dataSource:t.dataSource,file_id:e.id});await s.start(),await a()};await a(),P.delete(t.uid)}catch(t){throw H(t,{message:"run upload task failed",detail:"run upload task failed"})}};import{getSdk as N}from"@soga/sdk";import{RecordType as W}from"@soga/types";import{DownloadFile as C}from"@soga/entities";import{buildDError as Q}from"@soga/error";var K=async t=>{const{dataSource:e,sdk_domain:i,record_ids:a,space_id:s}=t,o=N(i),{name:r}=await o.getSpaceInfo({space_id:t.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:n}=l,d=e.getRepository(C);let c=[];try{const{length:t}=a,e=[];for(let i=0;i<t;i++){const t=a[i];i>0&&await new Promise(t=>setTimeout(t,500));const l=await o.getRecordInfo({space_id:s,record_id:t,refresh:!0});if(!l?.manifest)continue;const d={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:n,detail:l,file_total:l.type==W.FOLDER?void 0:1,is_ready:l.type!=W.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};e.push(d)}const i=e.filter(t=>t.type!=W.FOLDER);if(i.length){const t=i.length;for(let e=0;e<t;e++){const t=i[e],{info:a,...s}=t,o=(await d.save(d.create({...s})))[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=a;if(t.has_attachment){const t=r.manifest.attachments;for(const e of t){const{uuid:t,type:i,ftype:a,title:o}=e,r={...s,attachment_uuid:t,title:o,type:i,ftype:a},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}const l=e.filter(t=>t.type==W.FOLDER);if(l.length){const t=l.length;for(let e=0;e<t;e++){const t=l[e],{info:i,...a}=t,s=await d.save(d.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=i,n=r.manifest?.attachments;for(const t of n){const{uuid:e,type:i,ftype:s,title:o}=t,r={...a,attachment_uuid:e,has_attachment:!1,title:o,type:i,ftype:s},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}catch(t){try{c.length&&await d.delete(c)}catch(t){}throw Q(t,{message:"create download task failed",detail:"create download task failed"})}};import{buildDError as q}from"@soga/error";import{DownloadFile as G}from"@soga/entities";import{RecordType as J}from"@soga/types";import{getSdk as X}from"@soga/sdk";import{Not as Y}from"typeorm";var Z=async t=>{const{dataSource:e,sdk_domain:i,space_id:a,uid:s}=t,o=X(i),r=e.getRepository(G);try{const t=await r.findBy({uid:s,root_id:0,is_ready:!1}),e=async t=>{let i=1,s=1e3,l=[];const n=0==t.root_id?t.id:t.root_id;for(;s>=100;){const e=await o.getRecordList({space_id:a,parent_id:t.detail.id,page:i,pagesize:100});e?(s=e.length,l=l.concat(e),i++):s=0}if(l.length){const i=l.map(e=>({title:e.name,cloud_id:e.id,baidu_host_id:t.baidu_host_id,ali_host_id:t.ali_host_id,pid:t.id,type:e.type,ftype:e.ftype,routes:[...t.routes,e.name],root_id:n,space_id:e.space_id,space_name:t.space_name,uid:t.uid,detail:e,file_total:e.type==J.FOLDER?void 0:1})),a=i.filter(t=>t.type!=J.FOLDER);for(const t of a){await r.findOneBy({root_id:n,cloud_id:t.cloud_id})||await r.insert(t)}const s=i.filter(t=>t.type==J.FOLDER);for(const t of s){const i=await r.findOneBy({root_id:n,cloud_id:t.cloud_id});if(i)await e(i);else{const i=await r.save(r.create(t));await e(i)}}}};for(const i of t){await e(i);const t=await r.countBy({root_id:i.id,type:Y(J.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:t})}}catch(t){throw q(t,{message:"run download task failed",detail:"run download task failed"})}};export{K as createDownloadTask,w as createUploadTask,Z as runDownloadTask,M as runUploadTask};
+import{HostType as t,RecordFileKeep as e,RecordType as i}from"@soga/types";import{dirname as a,join as s,parse as o}from"path";import{getFileinfo as r,isSubtitle as l,isValidPath as n}from"@soga/fileutils";import{getSdk as d}from"@soga/sdk";import{UploadFile as c}from"@soga/entities";import{relative as f,sep as p}from"path";import _ from"xregexp";var h=(t,e)=>{const i=e.filter(e=>t.parent_path.startsWith(e.parent_path));if(!i.length)return null;const a=i.map(e=>{const i=f(e.parent_path,t.parent_path).split(p).filter(t=>!!t).length,a=e.keywords.filter(e=>t.filename.includes(e)).length/e.keywords.length;return{...e,percent:1e3*a/(10*i+1)}}).sort((t,e)=>e.percent-t.percent)[0];return a.percent<1?null:a},m=(t,e)=>{const i=[],a=[...t];let s=[...e];for(const t of s){const e=h(t,a);if(e){const{filepath:i}=e,s=a.find(t=>t.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(t.filepath))}else i.push(t)}return[a,i]},u=t=>{const e=t.split(_("[^\\p{L}0-9]","g")).filter(t=>!!t);return[...new Set(e)]};import{buildDError as y}from"@soga/error";var w=async f=>{const p=f.dataSource.getRepository(c);let _=[];try{const c=d(f.sdk_domain),h=await c.getUserInfo({refresh:!1}),{id:w,private_space_id:g}=h,{cloud_info:b,album:R}=await c.getRecordInfo({space_id:f.space_id,record_id:f.record_id,refresh:!0}),k={file_keeps:R.config.file_keeps},O=k.file_keeps==e.SOURCE||k.file_keeps==e.BOTH,E=k.file_keeps==e.PREVIEW||k.file_keeps==e.BOTH,{name:D}=await c.getSpaceInfo({space_id:f.space_id,refresh:!0}),F=R.id;let x=null,B=null;b[t.ALI]&&(x=b[t.ALI].id),b[t.BAIDU]&&(B=b[t.BAIDU].id);const{paths:v}=f,L=[],I=[],S=[],z=[],{length:T}=v;for(let t=0;t<T;t++){const e=v[t].replace(/[/\\]$/,"");if(!await n(e))throw y(new Error,{message:`Invalid file path: ${e}`,detail:`Invalid file path: ${e}`});{const t=await r(e),{record_type:n,record_ftype:d}=t,{dir:c,base:f,name:p}=o(e),_={type:n,ftype:d,title:p,filename:f,basepath:s(c,p),filepath:e,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:a(e),file_type:"file"};if(n==i.FOLDER)I.push(_);else{const t=n==i.VIDEO||n==i.AUDIO,a=l(e);if(t){const t=u(_.title);z.push({..._,file_type:"media",keywords:t})}else a?S.push({..._,file_type:"subtitle"}):L.push(_)}}}const[U,$]=m(z,S),A=[...O?S:$,...E?U:z,...L,...I].map(t=>{const e={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:f.is_attachment,filepath:t.filepath,space_id:f.space_id,space_name:D,is_shop_space:g!=f.space_id,aid:F,ali_host_id:x,baidu_host_id:B,config:k,is_paused:!1,is_ready:t.type!=i.FOLDER,type:t.type,ftype:t.ftype,task_record_id:f.record_id,uid:w,total_count:t.type==i.FOLDER?void 0:1};return"media"==t.file_type&&(e.external_texts=t.external_texts),e});if(!A.length)return;const V=A.filter(t=>t.type!=i.FOLDER);if(V.length){const{length:t}=V,e=100;for(let i=0;i<t;i+=e){const t=A.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}const j=A.filter(t=>t.type==i.FOLDER);if(j.length){const{length:t}=j,e=100;for(let i=0;i<t;i+=e){const t=j.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}}catch(t){try{_.length&&await p.delete(_)}catch(t){}throw y(t,{message:"create upload task failed",detail:`create upload task failed: [${f.paths.join(", ")}]`})}};import{UploadFile as g}from"@soga/entities";import{UploadFile as b}from"@soga/entities";import{RecordFileKeep as R,RecordFtype as k,RecordType as O}from"@soga/types";import{remove as E}from"fs-extra";import{glob as D}from"glob";import{homedir as F}from"os";import{resolve as x,dirname as B,parse as v,join as L,basename as I}from"path";import{Not as S}from"typeorm";import{getFileinfo as z,isSubtitle as T,isValidPath as U}from"@soga/fileutils";import{getDb as $}from"@soga/lowdb";import{createHash as A}from"crypto";import{buildDError as V}from"@soga/error";var j=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(t){const e=A("md5");return e.update(`${t}`),e.digest("hex")}constructor({dataSource:t,file_id:e}){this.dataSource=t,this.fileRepository=t.getRepository(b),this.file_id=e}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(t){throw V(t,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await this.fileRepository.countBy({root_id:this.file_id,type:S(O.FOLDER)});t.is_ready=!0,t.total_count=e,await this.fileRepository.save(t)}async insertFiles(){const{files_inserted:t,files:e,medias:i,subtitles:a}=this.db.data;if(t)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==R.SOURCE||o.file_keeps==R.BOTH),l=!!s.is_attachment||(o.file_keeps==R.PREVIEW||o.file_keeps==R.BOTH),[n,d]=m(i,a),c=[...r?a:d,...e,...l?n:i],{length:f}=c;if(!f)return;const p=[];for(let t=0;t<f;t++){const e=c[t],{filepath:i}=e;if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=B(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,n={is_attachment:s.is_attachment,input:{filename:e.filename,filepath:e.filepath,filesize:e.filesize,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime},filepath:e.filepath,space_id:s.space_id,space_name:s.space_name,is_shop_space:s.is_shop_space,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:e.type!=O.FOLDER,type:e.type,ftype:e.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==e.file_type&&(n.external_texts=e.external_texts),p.push(n)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const t=_.manager.getRepository(b),{length:e}=p,i=50;for(let a=0;a<e;a+=i){const e=p.slice(a,a+i);await t.createQueryBuilder().insert().into(b).values(e).execute()}await _.commitTransaction()}catch(t){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:t,folders_inserted:e}=this.db.data;if(e)return;const{length:i}=t,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let e=0;e<i;e++){const i=x(a.filepath,t[e]);if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=x(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,n={is_attachment:a.is_attachment,input:{filename:I(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:O.FOLDER,ftype:k.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(n))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await D("**/*",{nodir:!0,cwd:t.filepath});let i=[];e.map(t=>{let e=B(t);const a=[];for(;"."!=e&&!i.includes(e);)a.unshift(e),e=B(e);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=e;for(let i=0;i<r;i++){const r=x(t.filepath,e[i]);if(await U(r)){const t=await z(r),{record_type:e,record_ftype:i}=t,{dir:l,base:n,name:d}=v(r),c={type:e,ftype:i,title:d,filename:n,basepath:L(l,d),filepath:r,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:B(r),file_type:"file"},f=e==O.VIDEO||e==O.AUDIO,p=T(r);if(f){const t=u(c.title);o.push({...c,file_type:"media",keywords:t})}else p?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await E(this.cache_file)}async init(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const{id:e,input:i}=t,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=x(F(),".dpan","desktop","json_files",`${e}_${a}.json`);this.db=await $(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}};import{buildDError as H}from"@soga/error";var P=new Map,M=async t=>{try{const e=P.get(t.uid);if(e)return e;P.set(t.uid,!0);const i=t.dataSource.getRepository(g),a=async()=>{const e=await i.findOneBy({uid:t.uid,root_id:0,is_ready:!1});if(!e)return;const s=new j({dataSource:t.dataSource,file_id:e.id});await s.start(),await a()};await a(),P.delete(t.uid)}catch(t){throw H(t,{message:"run upload task failed",detail:"run upload task failed"})}};import{getSdk as N}from"@soga/sdk";import{RecordType as W}from"@soga/types";import{DownloadFile as C}from"@soga/entities";import{buildDError as Q}from"@soga/error";var K=async t=>{const{dataSource:e,sdk_domain:i,record_ids:a,space_id:s}=t,o=N(i),{name:r}=await o.getSpaceInfo({space_id:t.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:n}=l,d=e.getRepository(C);let c=[];try{const{length:t}=a,e=[];for(let i=0;i<t;i++){const t=a[i];i>0&&await new Promise(t=>setTimeout(t,500));const l=await o.getRecordInfo({space_id:s,record_id:t,refresh:!0}),d={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:n,detail:l,file_total:l.type==W.FOLDER?void 0:1,is_ready:l.type!=W.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};e.push(d)}const i=e.filter(t=>t.type!=W.FOLDER);if(i.length){const t=i.length;for(let e=0;e<t;e++){const t=i[e],{info:a,...s}=t,o=(await d.save(d.create({...s})))[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=a;if(t.has_attachment){const t=r.manifest.attachments;for(const e of t){const{uuid:t,type:i,ftype:a,title:o}=e,r={...s,attachment_uuid:t,title:o,type:i,ftype:a},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}const l=e.filter(t=>t.type==W.FOLDER);if(l.length){const t=l.length;for(let e=0;e<t;e++){const t=l[e],{info:i,...a}=t,s=await d.save(d.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=i,n=r.manifest?.attachments;for(const t of n){const{uuid:e,type:i,ftype:s,title:o}=t,r={...a,attachment_uuid:e,has_attachment:!1,title:o,type:i,ftype:s},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}catch(t){try{c.length&&await d.delete(c)}catch(t){}throw Q(t,{message:"create download task failed",detail:"create download task failed"})}};import{buildDError as q}from"@soga/error";import{DownloadFile as G}from"@soga/entities";import{RecordType as J}from"@soga/types";import{getSdk as X}from"@soga/sdk";import{Not as Y}from"typeorm";var Z=async t=>{const{dataSource:e,sdk_domain:i,space_id:a,uid:s}=t,o=X(i),r=e.getRepository(G);try{const t=await r.findBy({uid:s,root_id:0,is_ready:!1}),e=async t=>{let i=1,s=1e3,l=[];const n=0==t.root_id?t.id:t.root_id;for(;s>=100;){const e=await o.getRecordList({space_id:a,parent_id:t.detail.id,page:i,pagesize:100});e?(s=e.length,l=l.concat(e),i++):s=0}if(l.length){const i=l.map(e=>({title:e.name,cloud_id:e.id,baidu_host_id:t.baidu_host_id,ali_host_id:t.ali_host_id,pid:t.id,type:e.type,ftype:e.ftype,routes:[...t.routes,e.name],root_id:n,space_id:e.space_id,space_name:t.space_name,uid:t.uid,detail:e,file_total:e.type==J.FOLDER?void 0:1})),a=i.filter(t=>t.type!=J.FOLDER);for(const t of a){await r.findOneBy({root_id:n,cloud_id:t.cloud_id})||await r.insert(t)}const s=i.filter(t=>t.type==J.FOLDER);for(const t of s){const i=await r.findOneBy({root_id:n,cloud_id:t.cloud_id});if(i)await e(i);else{const i=await r.save(r.create(t));await e(i)}}}};for(const i of t){await e(i);const t=await r.countBy({root_id:i.id,type:Y(J.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:t})}}catch(t){throw q(t,{message:"run download task failed",detail:"run download task failed"})}};export{K as createDownloadTask,w as createUploadTask,Z as runDownloadTask,M as runUploadTask};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/task",
-  "version": "1.3.8",
+  "version": "1.3.12",
   "publishConfig": {
     "access": "public"
   },
@@ -17,7 +17,9 @@
     "tsup": "^8.5.0",
     "typescript": "^5.8.3",
     "@types/node": "^24.5.2",
-    "@soga/typescript-config": "1.3.8"
+    "reflect-metadata": "^0.2.2",
+    "better-sqlite3": "^12.4.6",
+    "@soga/typescript-config": "1.3.12"
   },
   "keywords": [],
   "author": "",
@@ -29,14 +31,14 @@
     "mime": "^4.0.7",
     "typeorm": "^0.3.24",
     "xregexp": "^5.1.2",
-    "@soga/entities": "1.3.8",
-    "@soga/utils": "1.3.8",
-    "@soga/sdk": "1.3.8",
-    "@soga/types": "1.3.8",
-    "@soga/node-types": "1.3.8",
-    "@soga/error": "1.3.8",
-    "@soga/fileutils": "1.3.8",
-    "@soga/lowdb": "1.3.8"
+    "@soga/entities": "1.3.12",
+    "@soga/fileutils": "1.3.12",
+    "@soga/sdk": "1.3.12",
+    "@soga/node-types": "1.3.12",
+    "@soga/types": "1.3.12",
+    "@soga/error": "1.3.12",
+    "@soga/utils": "1.3.12",
+    "@soga/lowdb": "1.3.12"
   },
   "scripts": {
     "demo": "tsx ./demo/demo.ts",