@soga/task 0.3.0 → 1.0.0

package/dist/main.d.mts

@@ -0,0 +1,37 @@
+import { DataSource } from 'typeorm';
+
+type UploadFileParams = {
+    dataSource: DataSource;
+    sdk_domain: string;
+    paths: string[];
+    space_id: number;
+    record_id: number;
+    is_attachment: boolean;
+};
+
+declare const createUploadTask: (params: UploadFileParams) => Promise<void>;
+
+declare const runUploadTask: (params: RunnerParams$1) => Promise<true>;
+type RunnerParams$1 = {
+    dataSource: DataSource;
+    uid: number;
+};
+
+type DownloadFileParams = {
+    dataSource: DataSource;
+    sdk_domain: string;
+    space_id: number;
+    record_ids: number[];
+};
+type RunnerParams = {
+    sdk_domain: string;
+    space_id: number;
+    dataSource: DataSource;
+    uid: number;
+};
+
+declare const createDownloadTask: (params: DownloadFileParams) => Promise<void>;
+
+declare const runDownloadTask: (params: RunnerParams) => Promise<void>;
+
+export { createDownloadTask, createUploadTask, runDownloadTask, runUploadTask };

package/dist/main.d.ts

@@ -1,3 +1,37 @@
-export * from './upload-task/create';
-export * from './upload-task/run';
-export * from './upload-task/create-affix';
+import { DataSource } from 'typeorm';
+
+type UploadFileParams = {
+    dataSource: DataSource;
+    sdk_domain: string;
+    paths: string[];
+    space_id: number;
+    record_id: number;
+    is_attachment: boolean;
+};
+
+declare const createUploadTask: (params: UploadFileParams) => Promise<void>;
+
+declare const runUploadTask: (params: RunnerParams$1) => Promise<true>;
+type RunnerParams$1 = {
+    dataSource: DataSource;
+    uid: number;
+};
+
+type DownloadFileParams = {
+    dataSource: DataSource;
+    sdk_domain: string;
+    space_id: number;
+    record_ids: number[];
+};
+type RunnerParams = {
+    sdk_domain: string;
+    space_id: number;
+    dataSource: DataSource;
+    uid: number;
+};
+
+declare const createDownloadTask: (params: DownloadFileParams) => Promise<void>;
+
+declare const runDownloadTask: (params: RunnerParams) => Promise<void>;
+
+export { createDownloadTask, createUploadTask, runDownloadTask, runUploadTask };

package/dist/main.js

@@ -1,19 +1 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./upload-task/create"), exports);
-__exportStar(require("./upload-task/run"), exports);
-__exportStar(require("./upload-task/create-affix"), exports);
+var e,t=Object.create,i=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,l=(e,t,o,l)=>{if(t&&"object"==typeof t||"function"==typeof t)for(let d of s(t))r.call(e,d)||d===o||i(e,d,{get:()=>t[d],enumerable:!(l=a(t,d))||l.enumerable});return e},d={};((e,t)=>{for(var a in t)i(e,a,{get:t[a],enumerable:!0})})(d,{createDownloadTask:()=>K,createUploadTask:()=>R,runDownloadTask:()=>C,runUploadTask:()=>j}),module.exports=(e=d,l(i({},"__esModule",{value:!0}),e));var n=require("@soga/types"),c=require("path"),p=require("@soga/fileutils"),f=require("@soga/sdk"),_=require("@soga/entities"),h=require("path"),u=((e,a,s)=>(s=null!=e?t(o(e)):{},l(!a&&e&&e.__esModule?s:i(s,"default",{value:e,enumerable:!0}),e)))(require("xregexp")),y=(e,t)=>{const i=t.filter(t=>e.parent_path.startsWith(t.parent_path));if(!i.length)return null;const a=i.map(t=>{const i=(0,h.relative)(t.parent_path,e.parent_path).split(h.sep).filter(e=>!!e).length,a=t.keywords.filter(t=>e.filename.includes(t)).length/t.keywords.length;return{...t,percent:1e3*a/(10*i+1)}}).sort((e,t)=>t.percent-e.percent)[0];return a.percent<1?null:a},m=(e,t)=>{const i=[],a=[...e];let s=[...t];for(const e of s){const t=y(e,a);if(t){const{filepath:i}=t,s=a.find(e=>e.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(e.filepath))}else i.push(e)}return[a,i]},w=e=>{const t=e.split((0,u.default)("[^\\p{L}0-9]","g")).filter(e=>!!e);return[...new Set(t)]},g=require("@soga/error"),R=async e=>{const t=e.dataSource.getRepository(_.UploadFile);let i=[];try{const a=(0,f.getSdk)(e.sdk_domain),s=await a.getUserInfo({refresh:!1}),{id:o}=s,{cloud_info:r,album:l}=await a.getRecordInfo({space_id:e.space_id,record_id:e.record_id,refresh:!0}),d={file_keeps:l.config.file_keeps},_=d.file_keeps==n.RecordFileKeep.SOURCE||d.file_keeps==n.RecordFileKeep.BOTH,h=d.file_keeps==n.RecordFileKeep.PREVIEW||d.file_keeps==n.RecordFileKeep.BOTH,{name:u}=await a.getSpaceInfo({space_id:e.space_id,refresh:!0}),y=l.id;let R=null,b=null;r[n.HostType.ALI]&&(R=r[n.HostType.ALI].id),r[n.HostType.BAIDU]&&(b=r[n.HostType.BAIDU].id);const{paths:O}=e,k=[],F=[],T=[],D=[],{length:E}=O;for(let e=0;e<E;e++){const t=O[e].replace(/[/\\]$/,"");if(!await(0,p.isValidPath)(t))throw(0,g.buildDError)(new Error,{message:`Invalid file path: ${t}`,detail:`Invalid file path: ${t}`});{const e=await(0,p.getFileinfo)(t),{record_type:i,record_ftype:a}=e,{dir:s,base:o,name:r}=(0,c.parse)(t),l={type:i,ftype:a,title:r,filename:o,basepath:(0,c.join)(s,r),filepath:t,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,c.dirname)(t),file_type:"file"};if(i==n.RecordType.FOLDER)F.push(l);else{const e=i==n.RecordType.VIDEO||i==n.RecordType.AUDIO,a=(0,p.isSubtitle)(t);if(e){const e=w(l.title);D.push({...l,file_type:"media",keywords:e})}else a?T.push({...l,file_type:"subtitle"}):k.push(l)}}}const[q,v]=m(D,T),x=[..._?T:v,...h?q:D,...k,...F].map(t=>{const i={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:e.is_attachment,filepath:t.filepath,space_id:e.space_id,space_name:u,aid:y,ali_host_id:R,baidu_host_id:b,config:d,is_paused:!1,is_ready:t.type!=n.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:e.record_id,uid:o,total_count:t.type==n.RecordType.FOLDER?void 0:1};return"media"==t.file_type&&(i.external_texts=t.external_texts),i});if(!x.length)return;const B=x.filter(e=>e.type!=n.RecordType.FOLDER);if(B.length){const{length:e}=B,a=100;for(let s=0;s<e;s+=a){const e=x.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}const L=x.filter(e=>e.type==n.RecordType.FOLDER);if(L.length){const{length:e}=L,a=100;for(let s=0;s<e;s+=a){const e=L.slice(s,s+a),o=await t.insert(e);i=i.concat(o.identifiers.map(e=>e.id))}}}catch(a){try{i.length&&await t.delete(i)}catch(e){}throw(0,g.buildDError)(a,{message:"create upload task failed",detail:`create upload task failed: [${e.paths.join(", ")}]`})}},b=require("@soga/entities"),O=require("@soga/entities"),k=require("@soga/types"),F=require("fs-extra"),T=require("glob"),D=require("os"),E=require("path"),q=require("typeorm"),v=require("@soga/fileutils"),x=require("@soga/lowdb"),B=require("crypto"),L=require("@soga/error"),S=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(e){const t=(0,B.createHash)("md5");return t.update(`${e}`),t.digest("hex")}constructor({dataSource:e,file_id:t}){this.dataSource=e,this.fileRepository=e.getRepository(O.UploadFile),this.file_id=t}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(e){throw(0,L.buildDError)(e,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await this.fileRepository.countBy({root_id:this.file_id,type:(0,q.Not)(k.RecordType.FOLDER)});e.is_ready=!0,e.total_count=t,await this.fileRepository.save(e)}async insertFiles(){const{files_inserted:e,files:t,medias:i,subtitles:a}=this.db.data;if(e)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.SOURCE||o.file_keeps==k.RecordFileKeep.BOTH),l=!!s.is_attachment||(o.file_keeps==k.RecordFileKeep.PREVIEW||o.file_keeps==k.RecordFileKeep.BOTH),[d,n]=m(i,a),c=[...r?a:n,...t,...l?d:i],{length:p}=c;if(!p)return;const f=[];for(let e=0;e<p;e++){const t=c[e],{filepath:i}=t;if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=(0,E.dirname)(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,d={is_attachment:s.is_attachment,input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},filepath:t.filepath,space_id:s.space_id,space_name:s.space_name,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:t.type!=k.RecordType.FOLDER,type:t.type,ftype:t.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==t.file_type&&(d.external_texts=t.external_texts),f.push(d)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const e=_.manager.getRepository(O.UploadFile),{length:t}=f,i=50;for(let a=0;a<t;a+=i){const t=f.slice(a,a+i);await e.createQueryBuilder().insert().into(O.UploadFile).values(t).execute()}await _.commitTransaction()}catch(e){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:e,folders_inserted:t}=this.db.data;if(t)return;const{length:i}=e,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let t=0;t<i;t++){const i=(0,E.resolve)(a.filepath,e[t]);if(!await(0,v.isValidPath)(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=(0,E.resolve)(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,d={is_attachment:a.is_attachment,input:{filename:(0,E.basename)(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:k.RecordType.FOLDER,ftype:k.RecordFtype.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(d))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const t=await(0,T.glob)("**/*",{nodir:!0,cwd:e.filepath});let i=[];t.map(e=>{let t=(0,E.dirname)(e);const a=[];for(;"."!=t&&!i.includes(t);)a.unshift(t),t=(0,E.dirname)(t);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=t;for(let i=0;i<r;i++){const r=(0,E.resolve)(e.filepath,t[i]);if(await(0,v.isValidPath)(r)){const e=await(0,v.getFileinfo)(r),{record_type:t,record_ftype:i}=e,{dir:l,base:d,name:n}=(0,E.parse)(r),c={type:t,ftype:i,title:n,filename:d,basepath:(0,E.join)(l,n),filepath:r,filesize:e.size,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime,parent_path:(0,E.dirname)(r),file_type:"file"},p=t==k.RecordType.VIDEO||t==k.RecordType.AUDIO,f=(0,v.isSubtitle)(r);if(p){const e=w(c.title);o.push({...c,file_type:"media",keywords:e})}else f?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await(0,F.remove)(this.cache_file)}async init(){const e=await this.fileRepository.findOneBy({id:this.file_id});if(!e)return;const{id:t,input:i}=e,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=(0,E.resolve)((0,D.homedir)(),".dpan","desktop","json_files",`${t}_${a}.json`);this.db=await(0,x.getDb)(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}},I=require("@soga/error"),U=new Map,j=async e=>{try{const t=U.get(e.uid);if(t)return t;U.set(e.uid,!0);const i=e.dataSource.getRepository(b.UploadFile),a=async()=>{const t=await i.findOneBy({uid:e.uid,root_id:0,is_ready:!1});if(!t)return;const s=new S({dataSource:e.dataSource,file_id:t.id});await s.start(),await a()};await a(),U.delete(e.uid)}catch(e){throw(0,I.buildDError)(e,{message:"run upload task failed",detail:"run upload task failed"})}},P=require("@soga/sdk"),z=require("@soga/types"),$=require("@soga/entities"),H=require("@soga/error"),K=async e=>{const{dataSource:t,sdk_domain:i,record_ids:a,space_id:s}=e,o=(0,P.getSdk)(i),{name:r}=await o.getSpaceInfo({space_id:e.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:d}=l,n=t.getRepository($.DownloadFile);let c=[];try{const{length:e}=a,t=[];for(let i=0;i<e;i++){const e=a[i];i>0&&await new Promise(e=>setTimeout(e,500));const l=await o.getRecordInfo({space_id:s,record_id:e,refresh:!0});if(!l?.manifest)continue;const n={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:d,detail:l,file_total:l.type==z.RecordType.FOLDER?void 0:1,is_ready:l.type!=z.RecordType.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};t.push(n)}const i=t.filter(e=>e.type!=z.RecordType.FOLDER);if(i.length){const e=i.length;for(let t=0;t<e;t++){const e=i[t],{info:a,...s}=e,o=(await n.save(n.create({...s})))[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=a;if(e.has_attachment){const e=r.manifest.attachments;for(const t of e){const{uuid:e,type:i,ftype:a,title:o}=t,r={...s,attachment_uuid:e,title:o,type:i,ftype:a},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}const l=t.filter(e=>e.type==z.RecordType.FOLDER);if(l.length){const e=l.length;for(let t=0;t<e;t++){const e=l[t],{info:i,...a}=e,s=await n.save(n.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!e.has_attachment)continue;const r=i,d=r.manifest?.attachments;for(const e of d){const{uuid:t,type:i,ftype:s,title:o}=e,r={...a,attachment_uuid:t,has_attachment:!1,title:o,type:i,ftype:s},l=await n.save(n.create(r));if(!l)continue;const d=l[0];d&&c.push(d.id)}}}}catch(e){try{c.length&&await n.delete(c)}catch(e){}throw(0,H.buildDError)(e,{message:"create download task failed",detail:"create download task failed"})}},V=require("@soga/error"),A=require("@soga/entities"),M=require("@soga/types"),N=require("@soga/sdk"),W=require("typeorm"),C=async e=>{const{dataSource:t,sdk_domain:i,space_id:a,uid:s}=e,o=(0,N.getSdk)(i),r=t.getRepository(A.DownloadFile);try{const e=await r.findBy({uid:s,root_id:0,is_ready:!1}),t=async e=>{let i=1,s=1e3,l=[];const d=0==e.root_id?e.id:e.root_id;for(;s>=100;){const t=await o.getRecordList({space_id:a,parent_id:e.detail.id,page:i,pagesize:100});t?(s=t.length,l=l.concat(t),i++):s=0}if(l.length){const i=l.map(t=>({title:t.name,cloud_id:t.id,baidu_host_id:e.baidu_host_id,ali_host_id:e.ali_host_id,pid:e.id,type:t.type,ftype:t.ftype,routes:[...e.routes,t.name],root_id:d,space_id:t.space_id,space_name:e.space_name,uid:e.uid,detail:t,file_total:t.type==M.RecordType.FOLDER?void 0:1})),a=i.filter(e=>e.type!=M.RecordType.FOLDER);for(const e of a){await r.findOneBy({root_id:d,cloud_id:e.cloud_id})||await r.insert(e)}const s=i.filter(e=>e.type==M.RecordType.FOLDER);for(const e of s){const i=await r.findOneBy({root_id:d,cloud_id:e.cloud_id});if(i)await t(i);else{const i=await r.save(r.create(e));await t(i)}}}};for(const i of e){await t(i);const e=await r.countBy({root_id:i.id,type:(0,W.Not)(M.RecordType.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:e})}}catch(e){throw(0,V.buildDError)(e,{message:"run download task failed",detail:"run download task failed"})}};

package/dist/main.mjs

@@ -0,0 +1 @@
+import{HostType as t,RecordFileKeep as e,RecordType as i}from"@soga/types";import{dirname as a,join as s,parse as o}from"path";import{getFileinfo as r,isSubtitle as l,isValidPath as n}from"@soga/fileutils";import{getSdk as d}from"@soga/sdk";import{UploadFile as c}from"@soga/entities";import{relative as f,sep as p}from"path";import _ from"xregexp";var h=(t,e)=>{const i=e.filter(e=>t.parent_path.startsWith(e.parent_path));if(!i.length)return null;const a=i.map(e=>{const i=f(e.parent_path,t.parent_path).split(p).filter(t=>!!t).length,a=e.keywords.filter(e=>t.filename.includes(e)).length/e.keywords.length;return{...e,percent:1e3*a/(10*i+1)}}).sort((t,e)=>e.percent-t.percent)[0];return a.percent<1?null:a},m=(t,e)=>{const i=[],a=[...t];let s=[...e];for(const t of s){const e=h(t,a);if(e){const{filepath:i}=e,s=a.find(t=>t.filepath==i);s&&(s.external_texts=s.external_texts||[],s.external_texts.push(t.filepath))}else i.push(t)}return[a,i]},u=t=>{const e=t.split(_("[^\\p{L}0-9]","g")).filter(t=>!!t);return[...new Set(e)]};import{buildDError as y}from"@soga/error";var w=async f=>{const p=f.dataSource.getRepository(c);let _=[];try{const c=d(f.sdk_domain),h=await c.getUserInfo({refresh:!1}),{id:w}=h,{cloud_info:g,album:b}=await c.getRecordInfo({space_id:f.space_id,record_id:f.record_id,refresh:!0}),R={file_keeps:b.config.file_keeps},k=R.file_keeps==e.SOURCE||R.file_keeps==e.BOTH,O=R.file_keeps==e.PREVIEW||R.file_keeps==e.BOTH,{name:E}=await c.getSpaceInfo({space_id:f.space_id,refresh:!0}),D=b.id;let F=null,x=null;g[t.ALI]&&(F=g[t.ALI].id),g[t.BAIDU]&&(x=g[t.BAIDU].id);const{paths:B}=f,L=[],v=[],I=[],S=[],{length:z}=B;for(let t=0;t<z;t++){const e=B[t].replace(/[/\\]$/,"");if(!await n(e))throw y(new Error,{message:`Invalid file path: ${e}`,detail:`Invalid file path: ${e}`});{const t=await r(e),{record_type:n,record_ftype:d}=t,{dir:c,base:f,name:p}=o(e),_={type:n,ftype:d,title:p,filename:f,basepath:s(c,p),filepath:e,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:a(e),file_type:"file"};if(n==i.FOLDER)v.push(_);else{const t=n==i.VIDEO||n==i.AUDIO,a=l(e);if(t){const t=u(_.title);S.push({..._,file_type:"media",keywords:t})}else a?I.push({..._,file_type:"subtitle"}):L.push(_)}}}const[T,U]=m(S,I),$=[...k?I:U,...O?T:S,...L,...v].map(t=>{const e={input:{filename:t.filename,filepath:t.filepath,filesize:t.filesize,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime},is_attachment:f.is_attachment,filepath:t.filepath,space_id:f.space_id,space_name:E,aid:D,ali_host_id:F,baidu_host_id:x,config:R,is_paused:!1,is_ready:t.type!=i.FOLDER,type:t.type,ftype:t.ftype,task_record_id:f.record_id,uid:w,total_count:t.type==i.FOLDER?void 0:1};return"media"==t.file_type&&(e.external_texts=t.external_texts),e});if(!$.length)return;const A=$.filter(t=>t.type!=i.FOLDER);if(A.length){const{length:t}=A,e=100;for(let i=0;i<t;i+=e){const t=$.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}const V=$.filter(t=>t.type==i.FOLDER);if(V.length){const{length:t}=V,e=100;for(let i=0;i<t;i+=e){const t=V.slice(i,i+e),a=await p.insert(t);_=_.concat(a.identifiers.map(t=>t.id))}}}catch(t){try{_.length&&await p.delete(_)}catch(t){}throw y(t,{message:"create upload task failed",detail:`create upload task failed: [${f.paths.join(", ")}]`})}};import{UploadFile as g}from"@soga/entities";import{UploadFile as b}from"@soga/entities";import{RecordFileKeep as R,RecordFtype as k,RecordType as O}from"@soga/types";import{remove as E}from"fs-extra";import{glob as D}from"glob";import{homedir as F}from"os";import{resolve as x,dirname as B,parse as L,join as v,basename as I}from"path";import{Not as S}from"typeorm";import{getFileinfo as z,isSubtitle as T,isValidPath as U}from"@soga/fileutils";import{getDb as $}from"@soga/lowdb";import{createHash as A}from"crypto";import{buildDError as V}from"@soga/error";var j=class{db;file_id;cache_file;dataSource;fileRepository;calculateMd5(t){const e=A("md5");return e.update(`${t}`),e.digest("hex")}constructor({dataSource:t,file_id:e}){this.dataSource=t,this.fileRepository=t.getRepository(b),this.file_id=e}async start(){try{await this.init(),await this.parseRawFiles(),await this.insertFolders(),await this.insertFiles(),await this.end(),await this.destroy()}catch(t){throw V(t,{message:"run upload task failed",detail:`run upload task failed, file_id: ${this.file_id}`})}}async end(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await this.fileRepository.countBy({root_id:this.file_id,type:S(O.FOLDER)});t.is_ready=!0,t.total_count=e,await this.fileRepository.save(t)}async insertFiles(){const{files_inserted:t,files:e,medias:i,subtitles:a}=this.db.data;if(t)return;const s=await this.fileRepository.findOneBy({id:this.file_id});if(!s)return;const{config:o}=s,r=!!s.is_attachment||(o.file_keeps==R.SOURCE||o.file_keeps==R.BOTH),l=!!s.is_attachment||(o.file_keeps==R.PREVIEW||o.file_keeps==R.BOTH),[n,d]=m(i,a),c=[...r?a:d,...e,...l?n:i],{length:f}=c;if(!f)return;const p=[];for(let t=0;t<f;t++){const e=c[t],{filepath:i}=e;if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const a=B(i),r=a==s.filepath?s:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:a});if(!r)continue;const l=r.id,n={is_attachment:s.is_attachment,input:{filename:e.filename,filepath:e.filepath,filesize:e.filesize,local_btime:e.local_btime,local_ctime:e.local_ctime,local_mtime:e.local_mtime},filepath:e.filepath,space_id:s.space_id,space_name:s.space_name,aid:s.aid,ali_host_id:s.ali_host_id,baidu_host_id:s.baidu_host_id,config:o,is_paused:!1,is_ready:e.type!=O.FOLDER,type:e.type,ftype:e.ftype,task_record_id:s.task_record_id,uid:s.uid,root_id:s.id,pid:l,total_count:1};"media"==e.file_type&&(n.external_texts=e.external_texts),p.push(n)}const _=this.dataSource.createQueryRunner();await _.connect(),await _.startTransaction();try{const t=_.manager.getRepository(b),{length:e}=p,i=50;for(let a=0;a<e;a+=i){const e=p.slice(a,a+i);await t.createQueryBuilder().insert().into(b).values(e).execute()}await _.commitTransaction()}catch(t){await _.rollbackTransaction()}finally{await _.release()}this.db.data.files_inserted=!0,await this.db.write()}async insertFolders(){const{folders:t,folders_inserted:e}=this.db.data;if(e)return;const{length:i}=t,a=await this.fileRepository.findOneBy({id:this.file_id});if(a){for(let e=0;e<i;e++){const i=x(a.filepath,t[e]);if(!await U(i))continue;if(await this.fileRepository.findOneBy({root_id:this.file_id,filepath:i}))continue;const s=x(i,".."),o=s==a.filepath?a:await this.fileRepository.findOneBy({root_id:this.file_id,filepath:s});if(!o)continue;const r=o.id,l=this.file_id,n={is_attachment:a.is_attachment,input:{filename:I(i),filepath:i,filesize:0,local_btime:0,local_ctime:0,local_mtime:0},aid:a.aid,ali_host_id:a.ali_host_id,baidu_host_id:a.baidu_host_id,config:a.config,filepath:i,is_ready:!1,uid:a.uid,type:O.FOLDER,ftype:k.NONE,task_record_id:a.task_record_id,space_name:a.space_name,space_id:a.space_id,is_paused:a.is_paused,external_texts:[],pid:r,root_id:l};await this.fileRepository.save(this.fileRepository.create(n))}this.db.data.folders_inserted=!0,await this.db.write()}}async parseRawFiles(){if(this.db.data.raw_files_parsed)return;const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const e=await D("**/*",{nodir:!0,cwd:t.filepath});let i=[];e.map(t=>{let e=B(t);const a=[];for(;"."!=e&&!i.includes(e);)a.unshift(e),e=B(e);i=[...i,...a]});const a=[],s=[],o=[],{length:r}=e;for(let i=0;i<r;i++){const r=x(t.filepath,e[i]);if(await U(r)){const t=await z(r),{record_type:e,record_ftype:i}=t,{dir:l,base:n,name:d}=L(r),c={type:e,ftype:i,title:d,filename:n,basepath:v(l,d),filepath:r,filesize:t.size,local_btime:t.local_btime,local_ctime:t.local_ctime,local_mtime:t.local_mtime,parent_path:B(r),file_type:"file"},f=e==O.VIDEO||e==O.AUDIO,p=T(r);if(f){const t=u(c.title);o.push({...c,file_type:"media",keywords:t})}else p?s.push({...c,file_type:"subtitle"}):a.push(c)}}this.db.data.folders=i,this.db.data.files=a,this.db.data.medias=o,this.db.data.subtitles=s,this.db.data.raw_files_parsed=!0,await this.db.write()}async destroy(){delete this.db,await E(this.cache_file)}async init(){const t=await this.fileRepository.findOneBy({id:this.file_id});if(!t)return;const{id:e,input:i}=t,a=this.calculateMd5(`${i.filepath}_${i.filesize}`);this.cache_file=x(F(),".dpan","desktop","json_files",`${e}_${a}.json`);this.db=await $(this.cache_file,{folders:[],subtitles:[],medias:[],files:[],status:{}})}};import{buildDError as H}from"@soga/error";var P=new Map,M=async t=>{try{const e=P.get(t.uid);if(e)return e;P.set(t.uid,!0);const i=t.dataSource.getRepository(g),a=async()=>{const e=await i.findOneBy({uid:t.uid,root_id:0,is_ready:!1});if(!e)return;const s=new j({dataSource:t.dataSource,file_id:e.id});await s.start(),await a()};await a(),P.delete(t.uid)}catch(t){throw H(t,{message:"run upload task failed",detail:"run upload task failed"})}};import{getSdk as N}from"@soga/sdk";import{RecordType as W}from"@soga/types";import{DownloadFile as C}from"@soga/entities";import{buildDError as Q}from"@soga/error";var K=async t=>{const{dataSource:e,sdk_domain:i,record_ids:a,space_id:s}=t,o=N(i),{name:r}=await o.getSpaceInfo({space_id:t.space_id,refresh:!0}),l=await o.getUserInfo({refresh:!1}),{id:n}=l,d=e.getRepository(C);let c=[];try{const{length:t}=a,e=[];for(let i=0;i<t;i++){const t=a[i];i>0&&await new Promise(t=>setTimeout(t,500));const l=await o.getRecordInfo({space_id:s,record_id:t,refresh:!0});if(!l?.manifest)continue;const d={title:l.name,cloud_id:l.id,baidu_host_id:l.cloud_info?.baidu?.id,ali_host_id:l.cloud_info?.ali?.id,type:l.type,ftype:l.ftype,routes:[l.name],root_id:0,space_id:l.space_id,space_name:r,uid:n,detail:l,file_total:l.type==W.FOLDER?void 0:1,is_ready:l.type!=W.FOLDER,has_attachment:!!l.manifest?.attachments?.length,info:l};e.push(d)}const i=e.filter(t=>t.type!=W.FOLDER);if(i.length){const t=i.length;for(let e=0;e<t;e++){const t=i[e],{info:a,...s}=t,o=(await d.save(d.create({...s})))[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=a;if(t.has_attachment){const t=r.manifest.attachments;for(const e of t){const{uuid:t,type:i,ftype:a,title:o}=e,r={...s,attachment_uuid:t,title:o,type:i,ftype:a},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}const l=e.filter(t=>t.type==W.FOLDER);if(l.length){const t=l.length;for(let e=0;e<t;e++){const t=l[e],{info:i,...a}=t,s=await d.save(d.create({...a}));if(!s)continue;const o=s[0];if(o&&c.push(o.id),!t.has_attachment)continue;const r=i,n=r.manifest?.attachments;for(const t of n){const{uuid:e,type:i,ftype:s,title:o}=t,r={...a,attachment_uuid:e,has_attachment:!1,title:o,type:i,ftype:s},l=await d.save(d.create(r));if(!l)continue;const n=l[0];n&&c.push(n.id)}}}}catch(t){try{c.length&&await d.delete(c)}catch(t){}throw Q(t,{message:"create download task failed",detail:"create download task failed"})}};import{buildDError as q}from"@soga/error";import{DownloadFile as G}from"@soga/entities";import{RecordType as J}from"@soga/types";import{getSdk as X}from"@soga/sdk";import{Not as Y}from"typeorm";var Z=async t=>{const{dataSource:e,sdk_domain:i,space_id:a,uid:s}=t,o=X(i),r=e.getRepository(G);try{const t=await r.findBy({uid:s,root_id:0,is_ready:!1}),e=async t=>{let i=1,s=1e3,l=[];const n=0==t.root_id?t.id:t.root_id;for(;s>=100;){const e=await o.getRecordList({space_id:a,parent_id:t.detail.id,page:i,pagesize:100});e?(s=e.length,l=l.concat(e),i++):s=0}if(l.length){const i=l.map(e=>({title:e.name,cloud_id:e.id,baidu_host_id:t.baidu_host_id,ali_host_id:t.ali_host_id,pid:t.id,type:e.type,ftype:e.ftype,routes:[...t.routes,e.name],root_id:n,space_id:e.space_id,space_name:t.space_name,uid:t.uid,detail:e,file_total:e.type==J.FOLDER?void 0:1})),a=i.filter(t=>t.type!=J.FOLDER);for(const t of a){await r.findOneBy({root_id:n,cloud_id:t.cloud_id})||await r.insert(t)}const s=i.filter(t=>t.type==J.FOLDER);for(const t of s){const i=await r.findOneBy({root_id:n,cloud_id:t.cloud_id});if(i)await e(i);else{const i=await r.save(r.create(t));await e(i)}}}};for(const i of t){await e(i);const t=await r.countBy({root_id:i.id,type:Y(J.FOLDER)});await r.update(i.id,{is_ready:!0,file_total:t})}}catch(t){throw q(t,{message:"run download task failed",detail:"run download task failed"})}};export{K as createDownloadTask,w as createUploadTask,Z as runDownloadTask,M as runUploadTask};

package/package.json

@@ -1,39 +1,46 @@
 {
   "name": "@soga/task",
-  "version": "0.3.0",
+  "version": "1.0.0",
   "publishConfig": {
     "access": "public"
   },
   "description": "",
-  "main": "dist/main.js",
-  "types": "dist/main.d.ts",
+  "main": "./dist/main.js",
+  "module": "./dist/main.mjs",
+  "types": "./dist/main.d.ts",
   "files": [
     "dist"
   ],
+  "scripts": {
+    "demo": "tsx ./demo/demo.ts",
+    "build": "rimraf dist && tsup src/main.ts --format cjs,esm --dts --minify terser",
+    "prepublishOnly": "npm run build"
+  },
   "devDependencies": {
-    "@types/mime": "^4.0.0"
+    "@soga/typescript-config": "latest",
+    "rimraf": "^6.0.1",
+    "terser": "^5.43.1",
+    "tsup": "^8.5.0",
+    "typescript": "^5.8.3",
+    "@types/node": "^24.5.2"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@soga/bridge": "^0.3.0",
-    "@soga/entities": "^0.3.0",
-    "@soga/sdk": "^0.3.0",
-    "@soga/types": "^0.3.0",
-    "@soga/utils": "^0.3.0",
+    "@soga/entities": "latest",
+    "@soga/fileutils": "latest",
+    "@soga/sdk": "latest",
+    "@soga/utils": "latest",
+    "@soga/types": "latest",
+    "@soga/node-types": "latest",
+    "@soga/error": "latest",
+    "@soga/lowdb": "latest",
+    "fs-extra": "^11.3.2",
     "axios": "^1.9.0",
+    "glob": "^11.0.2",
     "mime": "^4.0.7",
-    "typeorm": "^0.3.24"
-  },
-  "scripts": {
-    "build": "rimraf dist && tsc && ts-node ./scripts/minify",
-    "minify": "ts-node ./scripts/minify",
-    "demo_backup": "ts-node ./demo/demo.ts",
-    "demo": "ts-node ./demo/demo.ts",
-    "worker": "tsc && ts-node ./demo/worker.ts",
-    "test": "jest",
-    "dev": "ts-node ./src/main.ts",
-    "lint": "eslint . --ext .ts"
+    "typeorm": "^0.3.24",
+    "xregexp": "^5.1.2"
   }
-}
+}

package/dist/upload-task/create-affix.d.ts

@@ -1,2 +0,0 @@
-import { UploadFileParams } from './types';
-export declare const createUploadAffixTask: (params: UploadFileParams) => Promise<number>;

package/dist/upload-task/create-affix.js

@@ -1,78 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createUploadAffixTask = void 0;
-const types_1 = require("@soga/types");
-const fileinfo_1 = require("../utils/fileinfo");
-const path_1 = require("path");
-const sdk_1 = require("@soga/sdk");
-const entities_1 = require("@soga/entities");
-const createUploadAffixTask = async (params) => {
-    const sdk = (0, sdk_1.getSdk)(params.sdk_domain);
-    const userInfo = await sdk.getUserInfo({ refresh: false });
-    const { id: uid } = userInfo;
-    const { cloud_info, album } = await sdk.getRecordInfo({
-        space_id: params.space_id,
-        record_id: params.record_id,
-        refresh: true,
-    });
-    const config = {
-        file_keeps: album.config.file_keeps,
-    };
-    const { name: space_name } = await sdk.getSpaceInfo({
-        space_id: params.space_id,
-        refresh: true,
-    });
-    const aid = album.id;
-    let ali_host_id = 0;
-    let baidu_host_id = 0;
-    if (cloud_info[types_1.HostType.ALI]) {
-        ali_host_id = cloud_info[types_1.HostType.ALI].id;
-    }
-    if (cloud_info[types_1.HostType.BAIDU]) {
-        baidu_host_id = cloud_info[types_1.HostType.BAIDU].id;
-    }
-    const { paths } = params;
-    const { length } = paths;
-    const inputs = [];
-    for (let i = 0; i < length; i++) {
-        const filepath = paths[i];
-        const isValid = await (0, fileinfo_1.isValidPath)(filepath);
-        if (isValid) {
-            const fileinfo = await (0, fileinfo_1.getFileinfo)(filepath);
-            const { record_type } = fileinfo;
-            if (record_type == types_1.RecordType.FOLDER)
-                continue;
-            inputs.push({
-                filename: (0, path_1.basename)(filepath),
-                filepath,
-                filesize: fileinfo.size,
-                local_btime: fileinfo.local_btime,
-                local_ctime: fileinfo.local_ctime,
-                local_mtime: fileinfo.local_mtime,
-            });
-        }
-    }
-    const file = {
-        inputs,
-        space_id: params.space_id,
-        space_name,
-        filepath: '',
-        is_ready: true,
-        config,
-        uid,
-        aid,
-        baidu_host_id,
-        ali_host_id,
-        is_paused: false,
-        pid: 0,
-        root_id: 0,
-        total_count: 1,
-        type: types_1.RecordType.AFFIX,
-        task_record_id: params.record_id,
-        external_texts: [],
-    };
-    const fileRepository = params.dataSource.getRepository(entities_1.UploadFile);
-    const res = await fileRepository.save(fileRepository.create(file));
-    return res.id;
-};
-exports.createUploadAffixTask = createUploadAffixTask;

package/dist/upload-task/create.d.ts

@@ -1,2 +0,0 @@
-import { UploadFileParams } from './types';
-export declare const createUploadTask: (params: UploadFileParams) => Promise<void>;

package/dist/upload-task/create.js

@@ -1,165 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createUploadTask = void 0;
-const fileinfo_1 = require("../utils/fileinfo");
-const types_1 = require("@soga/types");
-const path_1 = require("path");
-const sdk_1 = require("@soga/sdk");
-const entities_1 = require("@soga/entities");
-const utils_1 = require("./utils");
-const createUploadTask = async (params) => {
-    const sdk = (0, sdk_1.getSdk)(params.sdk_domain);
-    const userInfo = await sdk.getUserInfo({ refresh: false });
-    const { id: uid } = userInfo;
-    const { cloud_info, album } = await sdk.getRecordInfo({
-        space_id: params.space_id,
-        record_id: params.record_id,
-        refresh: true,
-    });
-    const config = {
-        file_keeps: album.config.file_keeps,
-    };
-    const keep_source = config.file_keeps == types_1.RecordFileKeep.SOURCE ||
-        config.file_keeps == types_1.RecordFileKeep.BOTH;
-    const { name: space_name } = await sdk.getSpaceInfo({
-        space_id: params.space_id,
-        refresh: true,
-    });
-    const aid = album.id;
-    let ali_host_id = 0;
-    let baidu_host_id = 0;
-    if (cloud_info[types_1.HostType.ALI]) {
-        ali_host_id = cloud_info[types_1.HostType.ALI].id;
-    }
-    if (cloud_info[types_1.HostType.BAIDU]) {
-        baidu_host_id = cloud_info[types_1.HostType.BAIDU].id;
-    }
-    const { paths } = params;
-    const files = [];
-    const folders = [];
-    const subtitles = [];
-    const medias = [];
-    const { length } = paths;
-    for (let i = 0; i < length; i++) {
-        const filepath = paths[i];
-        const isValid = await (0, fileinfo_1.isValidPath)(filepath);
-        if (isValid) {
-            const fileinfo = await (0, fileinfo_1.getFileinfo)(filepath);
-            const { record_type } = fileinfo;
-            const { dir, base, name } = (0, path_1.parse)(filepath);
-            const info = {
-                type: record_type,
-                title: name,
-                filename: base,
-                basepath: (0, path_1.join)(dir, name),
-                filepath: filepath,
-                filesize: fileinfo.size,
-                local_btime: fileinfo.local_btime,
-                local_ctime: fileinfo.local_ctime,
-                local_mtime: fileinfo.local_mtime,
-                parent_path: (0, path_1.dirname)(filepath),
-                file_type: 'file',
-            };
-            if (record_type == types_1.RecordType.FOLDER) {
-                folders.push(info);
-            }
-            else {
-                const is_media = record_type == types_1.RecordType.VIDEO || record_type == types_1.RecordType.AUDIO;
-                const is_subtitle = (0, fileinfo_1.isSubtitle)(filepath);
-                if (is_media) {
-                    const keywords = info.title
-                        .split(/[ _\-.{}[\]、,，。：:!?<>《》【】]+/g)
-                        .filter((k) => k.length >= 2);
-                    medias.push({
-                        ...info,
-                        file_type: 'media',
-                        keywords,
-                    });
-                }
-                else if (is_subtitle) {
-                    subtitles.push({
-                        ...info,
-                        file_type: 'subtitle',
-                    });
-                }
-                else {
-                    files.push(info);
-                }
-            }
-        }
-    }
-    const match_map = {};
-    let subs = [...subtitles];
-    medias.forEach((media) => {
-        const matched = (0, utils_1.matchSubtitles)(media, subs);
-        if (matched) {
-            media.external_texts = matched.map((item) => item.filepath);
-            matched.forEach((item) => (match_map[item.filepath] = true));
-            subs = subs.filter((item) => !match_map[item.filepath]);
-        }
-    });
-    let subtitle_queues = subtitles;
-    if (!keep_source) {
-        subtitle_queues = subtitles.filter((item) => !match_map[item.filepath]);
-    }
-    const queues = [...subtitle_queues, ...medias, ...files, ...folders].map((item) => {
-        const info = {
-            inputs: [
-                {
-                    filename: item.filename,
-                    filepath: item.filepath,
-                    filesize: item.filesize,
-                    local_btime: item.local_btime,
-                    local_ctime: item.local_ctime,
-                    local_mtime: item.local_mtime,
-                },
-            ],
-            filepath: item.filepath,
-            space_id: params.space_id,
-            space_name,
-            aid,
-            ali_host_id,
-            baidu_host_id,
-            config,
-            is_paused: false,
-            is_ready: item.type != types_1.RecordType.FOLDER,
-            type: item.type,
-            task_record_id: params.record_id,
-            uid,
-            total_count: item.type == types_1.RecordType.FOLDER ? undefined : 1,
-        };
-        if (item.file_type == 'media') {
-            info.external_texts = item.external_texts;
-        }
-        return info;
-    });
-    if (!queues.length) {
-        return;
-    }
-    const queryRunner = params.dataSource.createQueryRunner();
-    await queryRunner.connect();
-    await queryRunner.startTransaction();
-    try {
-        const fileRepository = queryRunner.manager.getRepository(entities_1.UploadFile);
-        const { length } = queues;
-        const size = 50;
-        for (let i = 0; i < length; i += size) {
-            const items = queues.slice(i, i + size);
-            await fileRepository
-                .createQueryBuilder()
-                .insert()
-                .into(entities_1.UploadFile)
-                .values(items)
-                .execute();
-        }
-        await queryRunner.commitTransaction();
-    }
-    catch (err) {
-        await queryRunner.rollbackTransaction();
-        throw err;
-    }
-    finally {
-        await queryRunner.release();
-    }
-};
-exports.createUploadTask = createUploadTask;

package/dist/upload-task/run.d.ts

@@ -1,7 +0,0 @@
-import { DataSource } from 'typeorm';
-export declare const runUploadTask: (params: RunnerParams) => Promise<true>;
-type RunnerParams = {
-    dataSource: DataSource;
-    uid: number;
-};
-export {};

package/dist/upload-task/run.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runUploadTask = void 0;
-const entities_1 = require("@soga/entities");
-const single_runner_1 = require("./single-runner");
-const processMap = new Map();
-const runUploadTask = async (params) => {
-    const cache = processMap.get(params.uid);
-    if (cache)
-        return cache;
-    processMap.set(params.uid, true);
-    const fileRepository = params.dataSource.getRepository(entities_1.UploadFile);
-    const loop = async () => {
-        const fileitem = await fileRepository.findOneBy({
-            uid: params.uid,
-            root_id: 0,
-            is_ready: false,
-        });
-        if (!fileitem)
-            return;
-        const st = new single_runner_1.SingleRunner({
-            dataSource: params.dataSource,
-            file_id: fileitem.id,
-        });
-        await st.start();
-        await loop();
-    };
-    await loop();
-    processMap.delete(params.uid);
-};
-exports.runUploadTask = runUploadTask;

package/dist/upload-task/single-runner.d.ts

@@ -1,33 +0,0 @@
-import { DataSource } from 'typeorm';
-import { FileMeta, MediaMeta, SubtitleMeta } from './types';
-import { LowType } from '@soga/bridge';
-export declare class SingleRunner {
-    db: LowType<DbDataType>;
-    private file_id;
-    private cache_file;
-    private dataSource;
-    private fileRepository;
-    private calculateMd5;
-    constructor({ dataSource, file_id, }: {
-        dataSource: DataSource;
-        file_id: number;
-    });
-    start(): Promise<void>;
-    private end;
-    private insertFiles;
-    private insertFolders;
-    private parseRawFiles;
-    private destroy;
-    private init;
-}
-type DbDataType = {
-    raw_files_parsed: boolean;
-    folders_inserted: boolean;
-    files_inserted: boolean;
-    folders: string[];
-    subtitles: SubtitleMeta[];
-    medias: MediaMeta[];
-    files: FileMeta[];
-    status: Record<string, boolean>;
-};
-export {};

package/dist/upload-task/single-runner.js

@@ -1,336 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SingleRunner = void 0;
-const entities_1 = require("@soga/entities");
-const types_1 = require("@soga/types");
-const fs_extra_1 = require("fs-extra");
-const glob_1 = require("glob");
-const os_1 = require("os");
-const path_1 = require("path");
-const typeorm_1 = require("typeorm");
-const fileinfo_1 = require("../utils/fileinfo");
-const utils_1 = require("./utils");
-const bridge_1 = require("@soga/bridge");
-const crypto_1 = require("crypto");
-class SingleRunner {
-    db;
-    file_id;
-    cache_file;
-    dataSource;
-    fileRepository;
-    calculateMd5(text) {
-        const hash = (0, crypto_1.createHash)('md5');
-        hash.update(`${text}`);
-        return hash.digest('hex');
-    }
-    constructor({ dataSource, file_id, }) {
-        this.dataSource = dataSource;
-        this.fileRepository = dataSource.getRepository(entities_1.UploadFile);
-        this.file_id = file_id;
-    }
-    async start() {
-        await this.init();
-        await this.parseRawFiles();
-        await this.insertFolders();
-        await this.insertFiles();
-        await this.end();
-        await this.destroy();
-    }
-    async end() {
-        const fileitem = await this.fileRepository.findOneBy({
-            id: this.file_id,
-        });
-        if (!fileitem)
-            return;
-        const total = await this.fileRepository.countBy({
-            root_id: this.file_id,
-            type: (0, typeorm_1.Not)(types_1.RecordType.FOLDER),
-        });
-        fileitem.is_ready = true;
-        fileitem.total_count = total;
-        await this.fileRepository.save(fileitem);
-    }
-    async insertFiles() {
-        const { files_inserted, files: normal_files, medias, subtitles, } = this.db.data;
-        if (files_inserted)
-            return;
-        const fileitem = await this.fileRepository.findOneBy({
-            id: this.file_id,
-        });
-        if (!fileitem)
-            return;
-        const { config } = fileitem;
-        const keep_source = config.file_keeps == types_1.RecordFileKeep.SOURCE ||
-            config.file_keeps == types_1.RecordFileKeep.BOTH;
-        const match_map = {};
-        let subs = [...subtitles];
-        medias.forEach((media) => {
-            const matched = (0, utils_1.matchSubtitles)(media, subs);
-            if (matched) {
-                media.external_texts = matched.map((item) => item.filepath);
-                matched.forEach((item) => (match_map[item.filepath] = true));
-                subs = subs.filter((item) => !match_map[item.filepath]);
-            }
-        });
-        let subtitle_queues = subtitles;
-        if (!keep_source) {
-            subtitle_queues = subtitles.filter((item) => !match_map[item.filepath]);
-        }
-        const queues = [...subtitle_queues, ...normal_files, ...medias];
-        const { length } = queues;
-        if (!length)
-            return;
-        const items = [];
-        for (let i = 0; i < length; i++) {
-            const item = queues[i];
-            const { filepath } = item;
-            const isValid = await (0, fileinfo_1.isValidPath)(filepath);
-            if (!isValid)
-                continue;
-            const exist = await this.fileRepository.findOneBy({
-                root_id: this.file_id,
-                filepath,
-            });
-            if (exist)
-                continue;
-            const parentPath = (0, path_1.dirname)(filepath);
-            const parent = parentPath == fileitem.filepath
-                ? fileitem
-                : await this.fileRepository.findOneBy({
-                    root_id: this.file_id,
-                    filepath: parentPath,
-                });
-            if (!parent) {
-                continue;
-            }
-            const pid = parent.id;
-            const info = {
-                inputs: [
-                    {
-                        filename: item.filename,
-                        filepath: item.filepath,
-                        filesize: item.filesize,
-                        local_btime: item.local_btime,
-                        local_ctime: item.local_ctime,
-                        local_mtime: item.local_mtime,
-                    },
-                ],
-                filepath: item.filepath,
-                space_id: fileitem.space_id,
-                space_name: fileitem.space_name,
-                aid: fileitem.aid,
-                ali_host_id: fileitem.ali_host_id,
-                baidu_host_id: fileitem.baidu_host_id,
-                config,
-                is_paused: false,
-                is_ready: item.type != types_1.RecordType.FOLDER,
-                type: item.type,
-                task_record_id: fileitem.task_record_id,
-                uid: fileitem.uid,
-                root_id: fileitem.id,
-                pid,
-                total_count: 1,
-            };
-            if (item.file_type == 'media') {
-                info.external_texts = item.external_texts;
-            }
-            items.push(info);
-        }
-        const queryRunner = this.dataSource.createQueryRunner();
-        await queryRunner.connect();
-        await queryRunner.startTransaction();
-        try {
-            const fileRepository = queryRunner.manager.getRepository(entities_1.UploadFile);
-            const { length } = items;
-            const size = 50;
-            for (let i = 0; i < length; i += size) {
-                const chunks = items.slice(i, i + size);
-                await fileRepository
-                    .createQueryBuilder()
-                    .insert()
-                    .into(entities_1.UploadFile)
-                    .values(chunks)
-                    .execute();
-            }
-            await queryRunner.commitTransaction();
-        }
-        catch (err) {
-            await queryRunner.rollbackTransaction();
-        }
-        finally {
-            await queryRunner.release();
-        }
-        this.db.data.files_inserted = true;
-        await this.db.write();
-    }
-    async insertFolders() {
-        const { folders, folders_inserted } = this.db.data;
-        if (folders_inserted)
-            return;
-        const { length } = folders;
-        const fileitem = await this.fileRepository.findOneBy({
-            id: this.file_id,
-        });
-        if (!fileitem)
-            return;
-        for (let i = 0; i < length; i++) {
-            const folderPath = (0, path_1.resolve)(fileitem.filepath, folders[i]);
-            const isValid = await (0, fileinfo_1.isValidPath)(folderPath);
-            if (!isValid)
-                continue;
-            const exist = await this.fileRepository.findOneBy({
-                root_id: this.file_id,
-                filepath: folderPath,
-            });
-            if (exist)
-                continue;
-            const parentPath = (0, path_1.resolve)(folderPath, '..');
-            const parent = parentPath == fileitem.filepath
-                ? fileitem
-                : await this.fileRepository.findOneBy({
-                    root_id: this.file_id,
-                    filepath: parentPath,
-                });
-            if (!parent)
-                continue;
-            const pid = parent.id;
-            const root_id = this.file_id;
-            const info = {
-                inputs: [
-                    {
-                        filename: (0, path_1.basename)(folderPath),
-                        filepath: folderPath,
-                        filesize: 0,
-                        local_btime: 0,
-                        local_ctime: 0,
-                        local_mtime: 0,
-                    },
-                ],
-                aid: fileitem.aid,
-                ali_host_id: fileitem.ali_host_id,
-                baidu_host_id: fileitem.baidu_host_id,
-                config: fileitem.config,
-                filepath: folderPath,
-                is_ready: false,
-                uid: fileitem.uid,
-                type: types_1.RecordType.FOLDER,
-                task_record_id: fileitem.task_record_id,
-                space_name: fileitem.space_name,
-                space_id: fileitem.space_id,
-                is_paused: fileitem.is_paused,
-                external_texts: [],
-                pid,
-                root_id,
-            };
-            await this.fileRepository.save(this.fileRepository.create(info));
-        }
-        this.db.data.folders_inserted = true;
-        await this.db.write();
-    }
-    async parseRawFiles() {
-        if (this.db.data.raw_files_parsed) {
-            return;
-        }
-        const fileitem = await this.fileRepository.findOneBy({
-            id: this.file_id,
-        });
-        if (!fileitem)
-            return;
-        const paths = await (0, glob_1.glob)('**/*', {
-            nodir: true,
-            cwd: fileitem.filepath,
-        });
-        let folders = [];
-        paths.map((item) => {
-            let parentPath = (0, path_1.dirname)(item);
-            const arr = [];
-            while (parentPath != '.') {
-                if (!folders.includes(parentPath)) {
-                    arr.unshift(parentPath);
-                }
-                else {
-                    break;
-                }
-                parentPath = (0, path_1.dirname)(parentPath);
-            }
-            folders = [...folders, ...arr];
-        });
-        const files = [];
-        const subtitles = [];
-        const medias = [];
-        const { length } = paths;
-        for (let i = 0; i < length; i++) {
-            const filepath = (0, path_1.resolve)(fileitem.filepath, paths[i]);
-            const isValid = await (0, fileinfo_1.isValidPath)(filepath);
-            if (isValid) {
-                const fileinfo = await (0, fileinfo_1.getFileinfo)(filepath);
-                const { record_type } = fileinfo;
-                const { dir, base, name } = (0, path_1.parse)(filepath);
-                const info = {
-                    type: record_type,
-                    title: name,
-                    filename: base,
-                    basepath: (0, path_1.join)(dir, name),
-                    filepath: filepath,
-                    filesize: fileinfo.size,
-                    local_btime: fileinfo.local_btime,
-                    local_ctime: fileinfo.local_ctime,
-                    local_mtime: fileinfo.local_mtime,
-                    parent_path: (0, path_1.dirname)(filepath),
-                    file_type: 'file',
-                };
-                const is_media = record_type == types_1.RecordType.VIDEO || record_type == types_1.RecordType.AUDIO;
-                const is_subtitle = (0, fileinfo_1.isSubtitle)(filepath);
-                if (is_media) {
-                    const keywords = info.title
-                        .split(/[ _\-.{}[\]、,，。：:!?<>《》【】]+/g)
-                        .filter((k) => k.length >= 2);
-                    medias.push({
-                        ...info,
-                        file_type: 'media',
-                        keywords,
-                    });
-                }
-                else if (is_subtitle) {
-                    subtitles.push({
-                        ...info,
-                        file_type: 'subtitle',
-                    });
-                }
-                else {
-                    files.push(info);
-                }
-            }
-        }
-        this.db.data.folders = folders;
-        this.db.data.files = files;
-        this.db.data.medias = medias;
-        this.db.data.subtitles = subtitles;
-        this.db.data.raw_files_parsed = true;
-        await this.db.write();
-    }
-    async destroy() {
-        delete this.db;
-        await (0, fs_extra_1.remove)(this.cache_file);
-    }
-    async init() {
-        const fileitem = await this.fileRepository.findOneBy({
-            id: this.file_id,
-        });
-        if (!fileitem)
-            return;
-        const { id, inputs } = fileitem;
-        const input = inputs[0];
-        const md5 = this.calculateMd5(`${input.filepath}_${input.filesize}`);
-        this.cache_file = (0, path_1.resolve)((0, os_1.homedir)(), '.dpan', 'desktop', 'json_files', `${id}_${md5}.json`);
-        const defaultData = {
-            folders: [],
-            subtitles: [],
-            medias: [],
-            files: [],
-            status: {},
-        };
-        this.db = await (0, bridge_1.getDb)(this.cache_file, defaultData);
-    }
-}
-exports.SingleRunner = SingleRunner;

package/dist/upload-task/types.d.ts

@@ -1,53 +0,0 @@
-import { RecordType, UploadInputItem } from '@soga/types';
-import { DataSource } from 'typeorm';
-export type UploadFileParams = {
-    dataSource: DataSource;
-    sdk_domain: string;
-    paths: string[];
-    space_id: number;
-    record_id: number;
-};
-interface CommonMeta {
-    type: RecordType;
-    filename: string;
-    filepath: string;
-    filesize: number;
-    local_btime: number;
-    local_ctime: number;
-    local_mtime: number;
-    parent_path: string;
-    title: string;
-    basepath: string;
-    file_type: 'media' | 'subtitle' | 'folder' | 'file';
-}
-export interface FileMeta extends CommonMeta {
-    file_type: 'file';
-}
-export interface MediaMeta extends CommonMeta {
-    keywords: string[];
-    external_texts?: string[];
-    file_type: 'media';
-}
-export interface SubtitleMeta extends CommonMeta {
-    file_type: 'subtitle';
-}
-export interface FileItem {
-    inputs: UploadInputItem[];
-    space_id: number;
-    space_name: string;
-    filepath: string;
-    is_ready: boolean;
-    config: object;
-    uid: number;
-    aid: number;
-    baidu_host_id: number;
-    ali_host_id: number;
-    is_paused: boolean;
-    type: RecordType;
-    task_record_id: number;
-    external_texts: string[];
-    root_id?: number;
-    pid?: number;
-    total_count?: number;
-}
-export {};

package/dist/upload-task/types.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/upload-task/utils.d.ts

@@ -1,2 +0,0 @@
-import { MediaMeta, SubtitleMeta } from './types';
-export declare const matchSubtitles: (media: MediaMeta, subtitles: SubtitleMeta[]) => SubtitleMeta[];

package/dist/upload-task/utils.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.matchSubtitles = void 0;
-const path_1 = require("path");
-const matchSubtitles = (media, subtitles) => {
-    const first_match = subtitles.filter((item) => item.parent_path.includes(media.parent_path));
-    if (!first_match.length)
-        return null;
-    const { keywords } = media;
-    const matched = [];
-    first_match.forEach((item) => {
-        if (item.basepath == media.basepath) {
-            matched.unshift(item);
-        }
-        else {
-            const relative_uri = (0, path_1.relative)(media.parent_path, item.parent_path);
-            const level = relative_uri.split(path_1.sep).filter((i) => !!i).length;
-            if (level < 2) {
-                const matched_keywords = keywords.filter((keyword) => item.title.includes(keyword));
-                if (matched_keywords.length &&
-                    matched_keywords.join('').length / keywords.length > 0.15) {
-                    matched.push(item);
-                }
-            }
-        }
-    });
-    return matched;
-};
-exports.matchSubtitles = matchSubtitles;

package/dist/utils/fileinfo.d.ts

@@ -1,10 +0,0 @@
-import { RecordType } from '@soga/types';
-export declare const getFileinfo: (filepath: string) => Promise<{
-    local_ctime: number;
-    local_mtime: number;
-    local_btime: number;
-    record_type: RecordType;
-    size: number;
-}>;
-export declare const isSubtitle: (filepath: string) => boolean;
-export declare const isValidPath: (filepath: string) => Promise<boolean>;

package/dist/utils/fileinfo.js

@@ -1,108 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isValidPath = exports.isSubtitle = exports.getFileinfo = void 0;
-const types_1 = require("@soga/types");
-const mime_1 = __importDefault(require("mime"));
-const fs_extra_1 = require("fs-extra");
-const promises_1 = require("fs/promises");
-const path_1 = require("path");
-const isTextFile = async (filepath, length = 1024 * 5) => {
-    const fd = await (0, promises_1.open)(filepath, 'r');
-    for (let i = 0; i < length; i++) {
-        const buf = Buffer.alloc(1);
-        const { bytesRead } = await fd.read(buf, 0, 1, i);
-        const char = buf.toString().charCodeAt(0);
-        if (bytesRead === 0) {
-            await fd.close();
-            return true;
-        }
-        else if (bytesRead === 1 && char === 0) {
-            await fd.close();
-            return false;
-        }
-    }
-    await fd.close();
-    return true;
-};
-const getRecordType = async (filepath) => {
-    const mimetype = mime_1.default.getType(filepath);
-    if (!mimetype) {
-        return types_1.RecordType.OTHER;
-    }
-    if (mimetype.startsWith('video/')) {
-        if (filepath.endsWith('.ts')) {
-            if (await isTextFile(filepath)) {
-                return types_1.RecordType.OTHER;
-            }
-            else {
-                return types_1.RecordType.VIDEO;
-            }
-        }
-        else {
-            return types_1.RecordType.VIDEO;
-        }
-    }
-    else if (mimetype.startsWith('audio/')) {
-        return types_1.RecordType.AUDIO;
-    }
-    else if (mimetype.startsWith('image/')) {
-        return types_1.RecordType.IMAGE;
-    }
-    else if (mimetype.startsWith('text/')) {
-        return types_1.RecordType.TXT;
-    }
-    else if (mimetype.startsWith('application/vnd.rn-realmedia')) {
-        return types_1.RecordType.VIDEO;
-    }
-    return types_1.RecordType.OTHER;
-};
-const getFileinfo = async (filepath) => {
-    try {
-        const stats = await (0, fs_extra_1.stat)(filepath);
-        const isFolder = stats.isDirectory();
-        const record_type = isFolder
-            ? types_1.RecordType.FOLDER
-            : await getRecordType(filepath);
-        return {
-            local_ctime: Math.floor(stats.ctimeMs),
-            local_mtime: Math.floor(stats.mtimeMs),
-            local_btime: Math.floor(stats.birthtimeMs),
-            record_type,
-            size: isFolder ? 0 : stats.size,
-        };
-    }
-    catch (err) {
-        const now = Date.now();
-        return {
-            local_ctime: now,
-            local_mtime: now,
-            local_btime: now,
-            record_type: types_1.RecordType.OTHER,
-            size: 0,
-        };
-    }
-};
-exports.getFileinfo = getFileinfo;
-const isSubtitle = (filepath) => {
-    const parsed = (0, path_1.parse)(filepath);
-    const ext = parsed.ext;
-    const types = ['.srt', '.vtt', '.ass', '.ssa', '.sbv', '.lrc'];
-    if (types.includes(ext)) {
-        return true;
-    }
-    else {
-        return false;
-    }
-};
-exports.isSubtitle = isSubtitle;
-const isValidPath = async (filepath) => {
-    const exists = await (0, fs_extra_1.pathExists)(filepath);
-    if (!exists) {
-        return false;
-    }
-    return true;
-};
-exports.isValidPath = isValidPath;