@soga/fetcher 0.2.36 → 0.2.37

package/dist/baidu/fetcher.js

@@ -1 +1 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:r}){const i=this.getKey({fs_id:t,access_token:e,ua:r});try{for(;processMap[i];)await new Promise((e=>setTimeout(e,100)));processMap[i]=!0;const n=await this.urlRepository.findOne({where:{key:i,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:i}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:i,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[i]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:r}){const i=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:r,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},l=c-t+1,u={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!i.includes(a)){n=e;try{return u.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),u.end=Date.now(),{dlink:e,speed:1e3*l/(u.end-u.start)}}catch(e){i.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*l/(u.end-u.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:r,list:i}=s||{};if(0!=r)throw new Error(`获取下载链接失败: ${r}`);if(!i||!i.length)throw new Error("获取下载链接失败");return`${i[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,abort_controller:r,times:i=0,ua:n}){const o=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:n});if(!o)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(o,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":n,Accept:"application/octet-stream"},signal:r?.signal,responseType:"arraybuffer",timeout:t})).data}catch(c){if("AbortError"===c.name)return;const d=this.getKey({fs_id:t,access_token:e,ua:n});if(await this.urlRepository.increment({key:d,dlink:o},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,abort_controller:r,ua:n});throw c}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=e=>(instance||(instance=new BaiduFetcher(e)),instance);exports.getBaiduFetcher=getBaiduFetcher;
+"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:r}){const i=this.getKey({fs_id:t,access_token:e,ua:r});try{for(;processMap[i];)await new Promise((e=>setTimeout(e,100)));processMap[i]=!0;const n=await this.urlRepository.findOne({where:{key:i,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:i}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:i,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[i]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:r}){const i=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:r,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},l=c-t+1,u={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!i.includes(a)){n=e;try{return u.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),u.end=Date.now(),{dlink:e,speed:1e3*l/(u.end-u.start)}}catch(e){i.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*l/(u.end-u.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:r,list:i}=s||{};if(0!=r)throw new Error(`获取下载链接失败: ${r}`);if(!i||!i.length)throw new Error("获取下载链接失败");return`${i[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,abort_controller:r,times:i=0,ua:n}){const o=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:n});if(!o)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(o,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":n,Accept:"application/octet-stream"},signal:r?.signal,responseType:"arraybuffer",timeout:t})).data}catch(c){if(console.log("-------------------------------"),console.log(c.name,c.message),"AbortError"===c.name)return;const d=this.getKey({fs_id:t,access_token:e,ua:n});if(await this.urlRepository.increment({key:d,dlink:o},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,abort_controller:r,ua:n});throw c}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=e=>(instance||(instance=new BaiduFetcher(e)),instance);exports.getBaiduFetcher=getBaiduFetcher;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/fetcher",
-  "version": "0.2.36",
+  "version": "0.2.37",
   "publishConfig": {
     "access": "public"
   },
@@ -53,5 +53,5 @@
     "axios": "*",
     "typeorm": "*"
   },
-  "gitHead": "87d3b9a7aceee9bb2e2888cc0d2abd60d0046c42"
+  "gitHead": "18fe3c3c3e7671c78d561d8132d56f69a13a3323"
 }