@soga/fetcher 0.2.33 → 0.2.34

package/dist/ali/fetch-buffer.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchAliBuffer=fetchAliBuffer;const fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchAliBuffer(e){const t=calculateRangeData(e);return await parallelFetchBuffer(t,e.dataSource)}async function parallelFetchBuffer(e,t){const{ranges:a,threads:c}=e,r=[...a],n=async()=>{if(!r.length)return;const e=r.shift();if(!e)return;const a=(0,common_1.getKey)(e);if(!common_1.bufferCache[a]?.data){const c=(0,fetcher_1.getAliFetcher)(t),{access_token:r,drive_id:n,file_id:o}=e,f=await c.getArrayBuffer({access_token:r,drive_id:n,file_id:o,start:e.start,end:e.end,ua:e.ua});common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(f)}}await n()},o=[];for(let e=0;e<c;e++)o.push(n());await Promise.all(o);const f=a.map((e=>{const t=(0,common_1.getKey)(e);return common_1.bufferCache[t]?.data?common_1.bufferCache[t].data:null})),u=Buffer.concat(f);return a.forEach((e=>{const t=(0,common_1.getKey)(e);delete common_1.bufferCache[t]})),u}function calculateRangeData(e){const{start:t,end:a}=e,c=Math.max(a-t+1,0),r=Math.ceil(c/1572864),n=Math.ceil(c/r),o=[];for(let c=0;c<r;c++){const r=t+c*n,f=Math.min(a,r+n-1);o.push({...e,start:r,end:f})}return{ranges:o,threads:2}}
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchAliBuffer=fetchAliBuffer;const fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchAliBuffer(e){const t=calculateRangeData(e);return await parallelFetchBuffer(t,e.dataSource)}async function parallelFetchBuffer(e,t){const{ranges:a,threads:r}=e,c=[...a],n=async()=>{if(!c.length)return;const e=c.shift();if(!e)return;const a=(0,common_1.getKey)(e);if(!common_1.bufferCache[a]?.data){const r=(0,fetcher_1.getAliFetcher)(t),{access_token:c,drive_id:n,file_id:o}=e,f=await r.getArrayBuffer({access_token:c,drive_id:n,file_id:o,start:e.start,end:e.end,ua:e.ua,abort_controller:e.abort_controller});common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(f)}}await n()},o=[];for(let e=0;e<r;e++)o.push(n());await Promise.all(o);let f=!0;const u=a.map((e=>{const t=(0,common_1.getKey)(e);return common_1.bufferCache[t]?.data?common_1.bufferCache[t].data:(f=!1,null)}));if(f){const e=Buffer.concat(u);return a.forEach((e=>{const t=(0,common_1.getKey)(e);delete common_1.bufferCache[t]})),e}return null}function calculateRangeData(e){const{start:t,end:a}=e,r=Math.max(a-t+1,0),c=Math.ceil(r/1572864),n=Math.ceil(r/c),o=[];for(let r=0;r<c;r++){const c=t+r*n,f=Math.min(a,c+n-1);o.push({...e,start:c,end:f})}return{ranges:o,threads:2}}

package/dist/ali/fetcher.d.ts

@@ -19,6 +19,7 @@ export declare class AliFetcher {
         start: number;
         end: number;
         ua?: string;
+        abort_controller?: AbortController;
     }): Promise<ArrayBuffer>;
     private multipleGetArrayBuffer;
     private fetchDownloadInfo;

package/dist/ali/fetcher.js

@@ -1 +1 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliFetcher=exports.AliFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null;class AliFetcher{axios;domain="https://openapi.alipan.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:r}){const a=this.getKey({access_token:e,file_id:i,ua:r});try{for(;processMap[a];)await new Promise((e=>setTimeout(e,100)));processMap[a]=!0;const s=await this.urlRepository.findOne({where:{key:a,fail:(0,typeorm_1.LessThan)(2),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{fail:"ASC"}});if(s)return s.fail>1?(await this.urlRepository.delete({key:a}),""):s.dlink;const{url:o,expiration:n}=await this.fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:a,speed:100,expire:new Date(n).getTime()}),o;throw new Error("ali fetch download url failed")}finally{delete processMap[a]}}async getArrayBuffer(e){return await this.multipleGetArrayBuffer({...e,times:0})}async multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:s=0,ua:o}){const n=await this.getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:o});if(!n)throw new Error("获取下载链接失败");try{const e=a-r+1,t=Math.max(1e3*Math.ceil(e/1024/50),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${r}-${a}`,"User-Agent":o,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(l){const c=this.getKey({access_token:e,file_id:i,ua:o});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(s||0)<3)return await this.multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:(s||0)+1,ua:o});throw l}}async fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r}){const a={Authorization:`Bearer ${e}`,"User-Agent":r},{url:s,expiration:o}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:t,file_id:i},{headers:a});return{url:s,expiration:o}}getKey({access_token:e,file_id:t,ua:i}){const r=i?encodeURIComponent(i):"ua";return`ali-${encodeURIComponent(e)}-${r}-${t}`}}exports.AliFetcher=AliFetcher;const getAliFetcher=e=>(instance||(instance=new AliFetcher(e)),instance);exports.getAliFetcher=getAliFetcher;
+"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliFetcher=exports.AliFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null;class AliFetcher{axios;domain="https://openapi.alipan.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:r}){const a=this.getKey({access_token:e,file_id:i,ua:r});try{for(;processMap[a];)await new Promise((e=>setTimeout(e,100)));processMap[a]=!0;const s=await this.urlRepository.findOne({where:{key:a,fail:(0,typeorm_1.LessThan)(2),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{fail:"ASC"}});if(s)return s.fail>1?(await this.urlRepository.delete({key:a}),""):s.dlink;const{url:o,expiration:n}=await this.fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:a,speed:100,expire:new Date(n).getTime()}),o;throw new Error("ali fetch download url failed")}finally{delete processMap[a]}}async getArrayBuffer(e){return await this.multipleGetArrayBuffer({...e,times:0})}async multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,abort_controller:s,times:o=0,ua:n}){const l=await this.getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:n});if(!l)throw new Error("获取下载链接失败");try{const e=a-r+1,t=Math.max(1e3*Math.ceil(e/1024/50),3e3);return(await axios_1.default.get(l,{headers:{Range:`bytes=${r}-${a}`,"User-Agent":n,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t,signal:s?.signal})).data}catch(s){const c=this.getKey({access_token:e,file_id:i,ua:n});if(await this.urlRepository.increment({key:c,dlink:l},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(o||0)<3)return await this.multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:(o||0)+1,ua:n});throw s}}async fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r}){const a={Authorization:`Bearer ${e}`,"User-Agent":r},{url:s,expiration:o}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:t,file_id:i},{headers:a});return{url:s,expiration:o}}getKey({access_token:e,file_id:t,ua:i}){const r=i?encodeURIComponent(i):"ua";return`ali-${encodeURIComponent(e)}-${r}-${t}`}}exports.AliFetcher=AliFetcher;const getAliFetcher=e=>(instance||(instance=new AliFetcher(e)),instance);exports.getAliFetcher=getAliFetcher;

package/dist/ali/get-buffer.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliDownloadBuffer=exports.getAliPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getAliBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:i,part_md5:f,access_token:a,host_id:d,host_vip_type:l,drive_id:s,file_id:_,cache_folder:c},u)=>{const n=(0,common_1.getKey)({part_md5:f,start:r,end:o});let w=null;if(c){w=await(0,low_store_1.getLowStore)(c,u);const e=await w.readBuffer(n);if(e)return e}const p=await(0,fetch_buffer_1.fetchAliBuffer)({dataSource:e,record_id:t,start:r,end:o,total:i,access_token:a,host_id:d,host_vip_type:l,drive_id:s,file_id:_,part_md5:f});return w&&await w.writeBuffer(n,p),p},getAliPreviewBuffer=async e=>await getAliBuffer(e,"preview");exports.getAliPreviewBuffer=getAliPreviewBuffer;const getAliDownloadBuffer=async e=>await getAliBuffer(e,"download");exports.getAliDownloadBuffer=getAliDownloadBuffer;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliDownloadBuffer=exports.getAliPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getAliBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:i,part_md5:f,access_token:a,host_id:l,host_vip_type:_,drive_id:c,file_id:d,cache_folder:s,abort_controller:n},u)=>{const w=(0,common_1.getKey)({part_md5:f,start:r,end:o});let p=null;if(s){p=await(0,low_store_1.getLowStore)(s,u);const e=await p.readBuffer(w);if(e)return e}const B=await(0,fetch_buffer_1.fetchAliBuffer)({dataSource:e,record_id:t,start:r,end:o,total:i,access_token:a,host_id:l,host_vip_type:_,drive_id:c,file_id:d,part_md5:f,abort_controller:n});return p&&await p.writeBuffer(w,B),B},getAliPreviewBuffer=async e=>await getAliBuffer(e,"preview");exports.getAliPreviewBuffer=getAliPreviewBuffer;const getAliDownloadBuffer=async e=>await getAliBuffer(e,"download");exports.getAliDownloadBuffer=getAliDownloadBuffer;

package/dist/baidu/fetch-buffer.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchBaiduBuffer=fetchBaiduBuffer;const baidu_ua_1=require("@soga/baidu-ua"),fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchBaiduBuffer(t){const e=calculateRangeData(t);return await parallelFetchBuffer(e,t.dataSource)}async function parallelFetchBuffer(t,e){const{ranges:a,threads:r}=t,c=[...a],n=async()=>{if(!c.length)return;const t=c.shift();if(!t)return;const a=(0,common_1.getKey)(t);if(!common_1.bufferCache[a]?.data){const r=(0,fetcher_1.getBaiduFetcher)(e),{access_token:c,fs_id:n,host_id:o}=t,{ua:u,finishBaiduUA:f}=await(0,baidu_ua_1.getBaiduUA)(o,t.total),s=await r.getArrayBuffer({access_token:c,fs_id:n,start:t.start,end:t.end,ua:t.auto_ua?u:t.ua});f(),common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(s)}}await n()},o=[];for(let t=0;t<r;t++)o.push(n());await Promise.all(o);const u=a.map((t=>{const e=(0,common_1.getKey)(t);return common_1.bufferCache[e]?.data?common_1.bufferCache[e].data:null})),f=Buffer.concat(u);return a.forEach((t=>{const e=(0,common_1.getKey)(t);delete common_1.bufferCache[e]})),f}function calculateRangeData(t){const{start:e,end:a,total:r}=t,c=Math.max(a-e+1,0);if(2==t.host_vip_type){const r=Math.ceil(c/1572864),n=Math.ceil(c/r),o=[];for(let c=0;c<r;c++){const r=e+c*n,u=Math.min(a,r+n-1);o.push({...t,start:r,end:u,auto_ua:!1})}return{ranges:o,threads:2}}{const n=Math.ceil(c/131072),o=Math.ceil(c/n),u=[];for(let r=0;r<n;r++){const c=e+r*o,n=Math.min(a,c+o-1);u.push({...t,start:c,end:n,auto_ua:!0})}return{ranges:u,threads:Math.min(r<52428800?8:6,n)}}}
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchBaiduBuffer=fetchBaiduBuffer;const baidu_ua_1=require("@soga/baidu-ua"),fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchBaiduBuffer(t){const e=calculateRangeData(t);return await parallelFetchBuffer(e,t.dataSource)}async function parallelFetchBuffer(t,e){const{ranges:a,threads:r}=t,n=[...a],c=async()=>{if(!n.length)return;const t=n.shift();if(!t)return;const a=(0,common_1.getKey)(t);if(!common_1.bufferCache[a]?.data){const r=(0,fetcher_1.getBaiduFetcher)(e),{access_token:n,fs_id:c,host_id:o}=t,{ua:u,finishBaiduUA:f}=await(0,baidu_ua_1.getBaiduUA)(o,t.total),s=await r.getArrayBuffer({access_token:n,fs_id:c,start:t.start,end:t.end,ua:t.auto_ua?u:t.ua});if(f(),!s||0===s.byteLength)return;common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(s)}}await c()},o=[];for(let t=0;t<r;t++)o.push(c());await Promise.all(o);let u=!0;const f=a.map((t=>{const e=(0,common_1.getKey)(t);return common_1.bufferCache[e]?.data?common_1.bufferCache[e].data:(u=!1,null)}));if(u){const t=Buffer.concat(f);return a.forEach((t=>{const e=(0,common_1.getKey)(t);delete common_1.bufferCache[e]})),t}return null}function calculateRangeData(t){const{start:e,end:a,total:r}=t,n=Math.max(a-e+1,0);if(2==t.host_vip_type){const r=Math.ceil(n/1572864),c=Math.ceil(n/r),o=[];for(let n=0;n<r;n++){const r=e+n*c,u=Math.min(a,r+c-1);o.push({...t,start:r,end:u,auto_ua:!1})}return{ranges:o,threads:2}}{const c=Math.ceil(n/131072),o=Math.ceil(n/c),u=[];for(let r=0;r<c;r++){const n=e+r*o,c=Math.min(a,n+o-1);u.push({...t,start:n,end:c,auto_ua:!0})}return{ranges:u,threads:Math.min(r<52428800?8:6,c)}}}

package/dist/baidu/fetcher.d.ts

@@ -16,6 +16,7 @@ export declare class BaiduFetcher {
         fs_id: number;
         start: number;
         end: number;
+        abort_controller?: AbortController;
         ua?: string;
     }): Promise<ArrayBuffer>;
     getDownloadUrlByRange({ access_token, fs_id, start, end, ua, }: {

package/dist/baidu/fetcher.js

@@ -1 +1 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:i}){const r=this.getKey({fs_id:t,access_token:e,ua:i});try{for(;processMap[r];)await new Promise((e=>setTimeout(e,100)));processMap[r]=!0;const n=await this.urlRepository.findOne({where:{key:r,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:r}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:i});if(o)return await this.urlRepository.save({dlink:o,key:r,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[r]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:i}){const r=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:i,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},u=c-t+1,l={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!r.includes(a)){n=e;try{return l.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),l.end=Date.now(),{dlink:e,speed:1e3*u/(l.end-l.start)}}catch(e){r.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*u/(l.end-l.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:i,list:r}=s||{};if(0!=i)throw new Error(`获取下载链接失败: ${i}`);if(!r||!r.length)throw new Error("获取下载链接失败");return`${r[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:i=0,ua:r}){const n=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:r});if(!n)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":r,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(o){const c=this.getKey({fs_id:t,access_token:e,ua:r});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,ua:r});throw o}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=e=>(instance||(instance=new BaiduFetcher(e)),instance);exports.getBaiduFetcher=getBaiduFetcher;
+"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:r}){const i=this.getKey({fs_id:t,access_token:e,ua:r});try{for(;processMap[i];)await new Promise((e=>setTimeout(e,100)));processMap[i]=!0;const n=await this.urlRepository.findOne({where:{key:i,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:i}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:i,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[i]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:r}){const i=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:r,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},l=c-t+1,u={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!i.includes(a)){n=e;try{return u.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),u.end=Date.now(),{dlink:e,speed:1e3*l/(u.end-u.start)}}catch(e){i.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*l/(u.end-u.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:r,list:i}=s||{};if(0!=r)throw new Error(`获取下载链接失败: ${r}`);if(!i||!i.length)throw new Error("获取下载链接失败");return`${i[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,abort_controller:r,times:i=0,ua:n}){const o=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:n});if(!o)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(o,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":n,Accept:"application/octet-stream"},signal:r?.signal,responseType:"arraybuffer",timeout:t})).data}catch(c){if("AbortError"===c.name)return;const d=this.getKey({fs_id:t,access_token:e,ua:n});if(await this.urlRepository.increment({key:d,dlink:o},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,abort_controller:r,ua:n});throw c}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=e=>(instance||(instance=new BaiduFetcher(e)),instance);exports.getBaiduFetcher=getBaiduFetcher;

package/dist/baidu/get-buffer.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduDownloadBuffer=exports.getBaiduPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getBaiduBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:a,part_md5:f,access_token:i,host_id:u,host_vip_type:d,fs_id:s,cache_folder:c},_)=>{let n=null;const B=(0,common_1.getKey)({part_md5:f,start:r,end:o});if(c){n=await(0,low_store_1.getLowStore)(c,_);const e=await n.readBuffer(B);if(e)return e}const w=await(0,fetch_buffer_1.fetchBaiduBuffer)({dataSource:e,record_id:t,start:r,end:o,total:a,access_token:i,host_id:u,host_vip_type:d,fs_id:s,part_md5:f});return n&&await n.writeBuffer(B,w),w},getBaiduPreviewBuffer=async e=>await getBaiduBuffer(e,"preview");exports.getBaiduPreviewBuffer=getBaiduPreviewBuffer;const getBaiduDownloadBuffer=async e=>await getBaiduBuffer(e,"download");exports.getBaiduDownloadBuffer=getBaiduDownloadBuffer;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduDownloadBuffer=exports.getBaiduPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getBaiduBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:a,part_md5:f,access_token:i,host_id:u,host_vip_type:d,fs_id:s,cache_folder:c,abort_controller:_},n)=>{let B=null;const w=(0,common_1.getKey)({part_md5:f,start:r,end:o});if(c){B=await(0,low_store_1.getLowStore)(c,n);const e=await B.readBuffer(w);if(e)return e}const l=await(0,fetch_buffer_1.fetchBaiduBuffer)({dataSource:e,record_id:t,start:r,end:o,total:a,access_token:i,host_id:u,host_vip_type:d,fs_id:s,part_md5:f,abort_controller:_});return B&&await B.writeBuffer(w,l),l},getBaiduPreviewBuffer=async e=>await getBaiduBuffer(e,"preview");exports.getBaiduPreviewBuffer=getBaiduPreviewBuffer;const getBaiduDownloadBuffer=async e=>await getBaiduBuffer(e,"download");exports.getBaiduDownloadBuffer=getBaiduDownloadBuffer;

package/dist/common/types.d.ts

@@ -9,6 +9,7 @@ interface CommonBufferParams {
     host_id: number;
     host_vip_type: number;
     access_token: string;
+    abort_controller?: AbortController;
     ua?: string;
 }
 export interface BaiduBufferParams extends CommonBufferParams {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/fetcher",
-  "version": "0.2.33",
+  "version": "0.2.34",
   "publishConfig": {
     "access": "public"
   },
@@ -53,5 +53,5 @@
     "axios": "*",
     "typeorm": "*"
   },
-  "gitHead": "70d4fae5e473e2f40b7ced6ca434c7b29a527547"
+  "gitHead": "20386c24678a89bd4ede163429c560eef6ca9a06"
 }