@soga/fetcher 0.1.23 → 0.2.33

package/dist/ali/fetch-buffer.d.ts

@@ -0,0 +1,2 @@
+import { AliBufferParams } from '../common/types';
+export declare function fetchAliBuffer(params: AliBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/ali/fetch-buffer.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchAliBuffer=fetchAliBuffer;const fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchAliBuffer(e){const t=calculateRangeData(e);return await parallelFetchBuffer(t,e.dataSource)}async function parallelFetchBuffer(e,t){const{ranges:a,threads:c}=e,r=[...a],n=async()=>{if(!r.length)return;const e=r.shift();if(!e)return;const a=(0,common_1.getKey)(e);if(!common_1.bufferCache[a]?.data){const c=(0,fetcher_1.getAliFetcher)(t),{access_token:r,drive_id:n,file_id:o}=e,f=await c.getArrayBuffer({access_token:r,drive_id:n,file_id:o,start:e.start,end:e.end,ua:e.ua});common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(f)}}await n()},o=[];for(let e=0;e<c;e++)o.push(n());await Promise.all(o);const f=a.map((e=>{const t=(0,common_1.getKey)(e);return common_1.bufferCache[t]?.data?common_1.bufferCache[t].data:null})),u=Buffer.concat(f);return a.forEach((e=>{const t=(0,common_1.getKey)(e);delete common_1.bufferCache[t]})),u}function calculateRangeData(e){const{start:t,end:a}=e,c=Math.max(a-t+1,0),r=Math.ceil(c/1572864),n=Math.ceil(c/r),o=[];for(let c=0;c<r;c++){const r=t+c*n,f=Math.min(a,r+n-1);o.push({...e,start:r,end:f})}return{ranges:o,threads:2}}

package/dist/{ali-fetcher.d.ts → ali/fetcher.d.ts}

@@ -1,11 +1,11 @@
 import { AxiosInstance } from 'axios';
-import { Repository } from 'typeorm';
+import { DataSource, Repository } from 'typeorm';
 import { FetchUrl } from '@soga/entities';
 export declare class AliFetcher {
     axios: AxiosInstance;
     private domain;
     protected urlRepository: Repository<FetchUrl>;
-    constructor();
+    constructor(dataSource: DataSource);
     getDownloadUrl({ access_token, drive_id, file_id, ua, }: {
         access_token: string;
         drive_id: string;
@@ -24,4 +24,4 @@ export declare class AliFetcher {
     private fetchDownloadInfo;
     private getKey;
 }
-export declare const getAliFetcher: () => AliFetcher;
+export declare const getAliFetcher: (dataSource: DataSource) => AliFetcher;

package/dist/ali/fetcher.js

@@ -0,0 +1 @@
+"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliFetcher=exports.AliFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null;class AliFetcher{axios;domain="https://openapi.alipan.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:r}){const a=this.getKey({access_token:e,file_id:i,ua:r});try{for(;processMap[a];)await new Promise((e=>setTimeout(e,100)));processMap[a]=!0;const s=await this.urlRepository.findOne({where:{key:a,fail:(0,typeorm_1.LessThan)(2),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{fail:"ASC"}});if(s)return s.fail>1?(await this.urlRepository.delete({key:a}),""):s.dlink;const{url:o,expiration:n}=await this.fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r});if(o)return await this.urlRepository.save({dlink:o,key:a,speed:100,expire:new Date(n).getTime()}),o;throw new Error("ali fetch download url failed")}finally{delete processMap[a]}}async getArrayBuffer(e){return await this.multipleGetArrayBuffer({...e,times:0})}async multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:s=0,ua:o}){const n=await this.getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:o});if(!n)throw new Error("获取下载链接失败");try{const e=a-r+1,t=Math.max(1e3*Math.ceil(e/1024/50),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${r}-${a}`,"User-Agent":o,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(l){const c=this.getKey({access_token:e,file_id:i,ua:o});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(s||0)<3)return await this.multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:(s||0)+1,ua:o});throw l}}async fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r}){const a={Authorization:`Bearer ${e}`,"User-Agent":r},{url:s,expiration:o}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:t,file_id:i},{headers:a});return{url:s,expiration:o}}getKey({access_token:e,file_id:t,ua:i}){const r=i?encodeURIComponent(i):"ua";return`ali-${encodeURIComponent(e)}-${r}-${t}`}}exports.AliFetcher=AliFetcher;const getAliFetcher=e=>(instance||(instance=new AliFetcher(e)),instance);exports.getAliFetcher=getAliFetcher;

package/dist/ali/get-buffer.d.ts

@@ -0,0 +1,3 @@
+import { AliPreviewBufferParams } from '../common/types';
+export declare const getAliPreviewBuffer: (params: AliPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;
+export declare const getAliDownloadBuffer: (params: AliPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;

package/dist/ali/get-buffer.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliDownloadBuffer=exports.getAliPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getAliBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:i,part_md5:f,access_token:a,host_id:d,host_vip_type:l,drive_id:s,file_id:_,cache_folder:c},u)=>{const n=(0,common_1.getKey)({part_md5:f,start:r,end:o});let w=null;if(c){w=await(0,low_store_1.getLowStore)(c,u);const e=await w.readBuffer(n);if(e)return e}const p=await(0,fetch_buffer_1.fetchAliBuffer)({dataSource:e,record_id:t,start:r,end:o,total:i,access_token:a,host_id:d,host_vip_type:l,drive_id:s,file_id:_,part_md5:f});return w&&await w.writeBuffer(n,p),p},getAliPreviewBuffer=async e=>await getAliBuffer(e,"preview");exports.getAliPreviewBuffer=getAliPreviewBuffer;const getAliDownloadBuffer=async e=>await getAliBuffer(e,"download");exports.getAliDownloadBuffer=getAliDownloadBuffer;

package/dist/baidu/fetch-buffer.d.ts

@@ -0,0 +1,2 @@
+import { BaiduBufferParams } from '../common/types';
+export declare function fetchBaiduBuffer(params: BaiduBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/baidu/fetch-buffer.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.fetchBaiduBuffer=fetchBaiduBuffer;const baidu_ua_1=require("@soga/baidu-ua"),fetcher_1=require("./fetcher"),common_1=require("../common/common");async function fetchBaiduBuffer(t){const e=calculateRangeData(t);return await parallelFetchBuffer(e,t.dataSource)}async function parallelFetchBuffer(t,e){const{ranges:a,threads:r}=t,c=[...a],n=async()=>{if(!c.length)return;const t=c.shift();if(!t)return;const a=(0,common_1.getKey)(t);if(!common_1.bufferCache[a]?.data){const r=(0,fetcher_1.getBaiduFetcher)(e),{access_token:c,fs_id:n,host_id:o}=t,{ua:u,finishBaiduUA:f}=await(0,baidu_ua_1.getBaiduUA)(o,t.total),s=await r.getArrayBuffer({access_token:c,fs_id:n,start:t.start,end:t.end,ua:t.auto_ua?u:t.ua});f(),common_1.bufferCache[a]={updated:Date.now(),data:Buffer.from(s)}}await n()},o=[];for(let t=0;t<r;t++)o.push(n());await Promise.all(o);const u=a.map((t=>{const e=(0,common_1.getKey)(t);return common_1.bufferCache[e]?.data?common_1.bufferCache[e].data:null})),f=Buffer.concat(u);return a.forEach((t=>{const e=(0,common_1.getKey)(t);delete common_1.bufferCache[e]})),f}function calculateRangeData(t){const{start:e,end:a,total:r}=t,c=Math.max(a-e+1,0);if(2==t.host_vip_type){const r=Math.ceil(c/1572864),n=Math.ceil(c/r),o=[];for(let c=0;c<r;c++){const r=e+c*n,u=Math.min(a,r+n-1);o.push({...t,start:r,end:u,auto_ua:!1})}return{ranges:o,threads:2}}{const n=Math.ceil(c/131072),o=Math.ceil(c/n),u=[];for(let r=0;r<n;r++){const c=e+r*o,n=Math.min(a,c+o-1);u.push({...t,start:c,end:n,auto_ua:!0})}return{ranges:u,threads:Math.min(r<52428800?8:6,n)}}}

package/dist/{baidu-fetcher.d.ts → baidu/fetcher.d.ts}

@@ -1,11 +1,11 @@
 import { AxiosInstance } from 'axios';
-import { Repository } from 'typeorm';
+import { DataSource, Repository } from 'typeorm';
 import { FetchUrl } from '@soga/entities';
 export declare class BaiduFetcher {
     axios: AxiosInstance;
     private domain;
     protected urlRepository: Repository<FetchUrl>;
-    constructor();
+    constructor(dataSource: DataSource);
     getDownloadUrl({ access_token, fs_id, ua, }: {
         access_token: string;
         fs_id: number;
@@ -31,4 +31,4 @@ export declare class BaiduFetcher {
     private multipleGetArrayBuffer;
     private getKey;
 }
-export declare const getBaiduFetcher: () => BaiduFetcher;
+export declare const getBaiduFetcher: (dataSource: DataSource) => BaiduFetcher;

package/dist/baidu/fetcher.js

@@ -0,0 +1 @@
+"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e))),this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:i}){const r=this.getKey({fs_id:t,access_token:e,ua:i});try{for(;processMap[r];)await new Promise((e=>setTimeout(e,100)));processMap[r]=!0;const n=await this.urlRepository.findOne({where:{key:r,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:r}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:i});if(o)return await this.urlRepository.save({dlink:o,key:r,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[r]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:i}){const r=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:i,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},u=c-t+1,l={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!r.includes(a)){n=e;try{return l.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),l.end=Date.now(),{dlink:e,speed:1e3*u/(l.end-l.start)}}catch(e){r.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*u/(l.end-l.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:i,list:r}=s||{};if(0!=i)throw new Error(`获取下载链接失败: ${i}`);if(!r||!r.length)throw new Error("获取下载链接失败");return`${r[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:i=0,ua:r}){const n=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:r});if(!n)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":r,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(o){const c=this.getKey({fs_id:t,access_token:e,ua:r});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,ua:r});throw o}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=e=>(instance||(instance=new BaiduFetcher(e)),instance);exports.getBaiduFetcher=getBaiduFetcher;

package/dist/baidu/get-buffer.d.ts

@@ -0,0 +1,3 @@
+import { BaiduPreviewBufferParams } from '../common/types';
+export declare const getBaiduPreviewBuffer: (params: BaiduPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;
+export declare const getBaiduDownloadBuffer: (params: BaiduPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;

package/dist/baidu/get-buffer.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduDownloadBuffer=exports.getBaiduPreviewBuffer=void 0;const common_1=require("../common/common"),fetch_buffer_1=require("./fetch-buffer"),low_store_1=require("../common/low-store"),getBaiduBuffer=async({dataSource:e,record_id:t,start:r,end:o,total:a,part_md5:f,access_token:i,host_id:u,host_vip_type:d,fs_id:s,cache_folder:c},_)=>{let n=null;const B=(0,common_1.getKey)({part_md5:f,start:r,end:o});if(c){n=await(0,low_store_1.getLowStore)(c,_);const e=await n.readBuffer(B);if(e)return e}const w=await(0,fetch_buffer_1.fetchBaiduBuffer)({dataSource:e,record_id:t,start:r,end:o,total:a,access_token:i,host_id:u,host_vip_type:d,fs_id:s,part_md5:f});return n&&await n.writeBuffer(B,w),w},getBaiduPreviewBuffer=async e=>await getBaiduBuffer(e,"preview");exports.getBaiduPreviewBuffer=getBaiduPreviewBuffer;const getBaiduDownloadBuffer=async e=>await getBaiduBuffer(e,"download");exports.getBaiduDownloadBuffer=getBaiduDownloadBuffer;

package/dist/{common.d.ts → common/common.d.ts}

@@ -2,9 +2,8 @@ export declare const bufferCache: Record<string, {
     updated: number;
     data: Buffer<ArrayBuffer>;
 }>;
-export declare const getKey: ({ record_id, part_name, start, end, }: {
-    record_id: number;
-    part_name: string;
+export declare const getKey: ({ part_md5, start, end, }: {
+    part_md5: string;
     start: number;
     end: number;
 }) => string;

package/dist/{common.js → common/common.js}

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getKey=exports.bufferCache=void 0,exports.bufferCache={};const getKey=({record_id:e,part_name:t,start:r,end:o})=>`${t}_${e}-${r}-${o}`;exports.getKey=getKey;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getKey=exports.bufferCache=void 0,exports.bufferCache={};const getKey=({part_md5:e,start:t,end:r})=>`${e}-${t}-${r}`;exports.getKey=getKey;

package/dist/common/low-store.d.ts

@@ -0,0 +1,19 @@
+import { LowType } from '@soga/bridge';
+import { CacheType } from './types';
+export declare class LowStore {
+    folder_path: string;
+    cache_path: string;
+    db: LowType<DbDataType>;
+    cache_type: CacheType;
+    constructor(folder_path: string, cache_type: CacheType);
+    init(): Promise<void>;
+    read(key: string): Promise<boolean>;
+    write(key: string, value: boolean): Promise<void>;
+    delete(key: string): Promise<void>;
+    writeBuffer(key: string, buffer: Buffer<ArrayBuffer | ArrayBufferLike>): Promise<void>;
+    readBuffer(key: string): Promise<Buffer<ArrayBufferLike>>;
+    deleteBuffer(key: string): Promise<void>;
+}
+export declare const getLowStore: (folder_path: string, cache_type: CacheType) => Promise<LowStore>;
+type DbDataType = Record<string, boolean>;
+export {};

package/dist/common/low-store.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getLowStore=exports.LowStore=void 0;const bridge_1=require("@soga/bridge"),fs_extra_1=require("fs-extra"),path_1=require("path");class LowStore{folder_path;cache_path;db;cache_type;constructor(t,e){this.folder_path=t,this.cache_type=e,this.cache_path=(0,path_1.resolve)(t,`${e}_cache`)}async init(){const t=(0,path_1.resolve)(this.cache_path,"cache.json");this.db=await(0,bridge_1.getDb)(t,{})}async read(t){return this.db.data[t]}async write(t,e){this.db.data[t]=e,await this.db.write()}async delete(t){delete this.db.data[t],await this.db.write()}async writeBuffer(t,e){const a=(0,path_1.resolve)(this.cache_path,t);await(0,fs_extra_1.writeFile)(a,e),this.db.data[t]=!0,await this.db.write()}async readBuffer(t){if(!this.db.data[t])return null;const e=(0,path_1.resolve)(this.cache_path,t);return await(0,fs_extra_1.readFile)(e)}async deleteBuffer(t){if(!this.db.data[t])return;delete this.db.data[t],await this.db.write();const e=(0,path_1.resolve)(this.cache_path,t);try{await(0,fs_extra_1.remove)(e)}catch(t){}}}exports.LowStore=LowStore;const storeMap={};let lastCleanTime=0;const getLowStore=async(t,e)=>{if(storeMap[t]){for(;!storeMap[t].inited;)await new Promise((t=>setTimeout(t,20)));storeMap[t].updated_at=Date.now();const e=Date.now();return e-lastCleanTime>18e5&&(lastCleanTime=e,Object.keys(storeMap).forEach((t=>{e-storeMap[t].updated_at>288e5&&delete storeMap[t]}))),storeMap[t].store}const a=new LowStore(t,e);return storeMap[t]={updated_at:Date.now(),store:a,inited:!1},await a.init(),storeMap[t].inited=!0,a};exports.getLowStore=getLowStore;

package/dist/{types.d.ts → common/types.d.ts}

@@ -1,6 +1,8 @@
+import { DataSource } from 'typeorm';
 interface CommonBufferParams {
+    dataSource: DataSource;
     record_id: number;
-    part_name: string;
+    part_md5: string;
     start: number;
     end: number;
     total: number;
@@ -17,11 +19,10 @@ export interface AliBufferParams extends CommonBufferParams {
     file_id: string;
 }
 export interface BaiduPreviewBufferParams extends BaiduBufferParams {
-    cache: boolean;
     cache_folder: string;
 }
 export interface AliPreviewBufferParams extends AliBufferParams {
-    cache: boolean;
     cache_folder: string;
 }
+export type CacheType = 'preview' | 'download';
 export {};

package/dist/main.d.ts

@@ -1,7 +1,7 @@
-export { BaiduFetcher, getBaiduFetcher } from './baidu-fetcher';
-export { AliFetcher, getAliFetcher } from './ali-fetcher';
-export { getBaiduBuffer } from './baidu-buffer';
-export { getAliBuffer } from './ali-buffer';
-export { getBaiduPreviewBuffer, getAliPreviewBuffer } from './preview-buffer';
-export { FetchUrl } from '@soga/entities';
-export * from './types';
+export { BaiduFetcher, getBaiduFetcher } from './baidu/fetcher';
+export { AliFetcher, getAliFetcher } from './ali/fetcher';
+export { fetchBaiduBuffer } from './baidu/fetch-buffer';
+export { fetchAliBuffer } from './ali/fetch-buffer';
+export { getBaiduPreviewBuffer, getBaiduDownloadBuffer, } from './baidu/get-buffer';
+export { getAliPreviewBuffer, getAliDownloadBuffer } from './ali/get-buffer';
+export * from './common/types';

package/dist/main.js

@@ -1 +1 @@
-"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,r,t,i){void 0===i&&(i=t);var u=Object.getOwnPropertyDescriptor(r,t);u&&!("get"in u?!r.__esModule:u.writable||u.configurable)||(u={enumerable:!0,get:function(){return r[t]}}),Object.defineProperty(e,i,u)}:function(e,r,t,i){void 0===i&&(i=t),e[i]=r[t]}),__exportStar=this&&this.__exportStar||function(e,r){for(var t in e)"default"===t||Object.prototype.hasOwnProperty.call(r,t)||__createBinding(r,e,t)};Object.defineProperty(exports,"__esModule",{value:!0}),exports.FetchUrl=exports.getAliPreviewBuffer=exports.getBaiduPreviewBuffer=exports.getAliBuffer=exports.getBaiduBuffer=exports.getAliFetcher=exports.AliFetcher=exports.getBaiduFetcher=exports.BaiduFetcher=void 0;var baidu_fetcher_1=require("./baidu-fetcher");Object.defineProperty(exports,"BaiduFetcher",{enumerable:!0,get:function(){return baidu_fetcher_1.BaiduFetcher}}),Object.defineProperty(exports,"getBaiduFetcher",{enumerable:!0,get:function(){return baidu_fetcher_1.getBaiduFetcher}});var ali_fetcher_1=require("./ali-fetcher");Object.defineProperty(exports,"AliFetcher",{enumerable:!0,get:function(){return ali_fetcher_1.AliFetcher}}),Object.defineProperty(exports,"getAliFetcher",{enumerable:!0,get:function(){return ali_fetcher_1.getAliFetcher}});var baidu_buffer_1=require("./baidu-buffer");Object.defineProperty(exports,"getBaiduBuffer",{enumerable:!0,get:function(){return baidu_buffer_1.getBaiduBuffer}});var ali_buffer_1=require("./ali-buffer");Object.defineProperty(exports,"getAliBuffer",{enumerable:!0,get:function(){return ali_buffer_1.getAliBuffer}});var preview_buffer_1=require("./preview-buffer");Object.defineProperty(exports,"getBaiduPreviewBuffer",{enumerable:!0,get:function(){return preview_buffer_1.getBaiduPreviewBuffer}}),Object.defineProperty(exports,"getAliPreviewBuffer",{enumerable:!0,get:function(){return preview_buffer_1.getAliPreviewBuffer}});var entities_1=require("@soga/entities");Object.defineProperty(exports,"FetchUrl",{enumerable:!0,get:function(){return entities_1.FetchUrl}}),__exportStar(require("./types"),exports);
+"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,r,t,f){void 0===f&&(f=t);var u=Object.getOwnPropertyDescriptor(r,t);u&&!("get"in u?!r.__esModule:u.writable||u.configurable)||(u={enumerable:!0,get:function(){return r[t]}}),Object.defineProperty(e,f,u)}:function(e,r,t,f){void 0===f&&(f=t),e[f]=r[t]}),__exportStar=this&&this.__exportStar||function(e,r){for(var t in e)"default"===t||Object.prototype.hasOwnProperty.call(r,t)||__createBinding(r,e,t)};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliDownloadBuffer=exports.getAliPreviewBuffer=exports.getBaiduDownloadBuffer=exports.getBaiduPreviewBuffer=exports.fetchAliBuffer=exports.fetchBaiduBuffer=exports.getAliFetcher=exports.AliFetcher=exports.getBaiduFetcher=exports.BaiduFetcher=void 0;var fetcher_1=require("./baidu/fetcher");Object.defineProperty(exports,"BaiduFetcher",{enumerable:!0,get:function(){return fetcher_1.BaiduFetcher}}),Object.defineProperty(exports,"getBaiduFetcher",{enumerable:!0,get:function(){return fetcher_1.getBaiduFetcher}});var fetcher_2=require("./ali/fetcher");Object.defineProperty(exports,"AliFetcher",{enumerable:!0,get:function(){return fetcher_2.AliFetcher}}),Object.defineProperty(exports,"getAliFetcher",{enumerable:!0,get:function(){return fetcher_2.getAliFetcher}});var fetch_buffer_1=require("./baidu/fetch-buffer");Object.defineProperty(exports,"fetchBaiduBuffer",{enumerable:!0,get:function(){return fetch_buffer_1.fetchBaiduBuffer}});var fetch_buffer_2=require("./ali/fetch-buffer");Object.defineProperty(exports,"fetchAliBuffer",{enumerable:!0,get:function(){return fetch_buffer_2.fetchAliBuffer}});var get_buffer_1=require("./baidu/get-buffer");Object.defineProperty(exports,"getBaiduPreviewBuffer",{enumerable:!0,get:function(){return get_buffer_1.getBaiduPreviewBuffer}}),Object.defineProperty(exports,"getBaiduDownloadBuffer",{enumerable:!0,get:function(){return get_buffer_1.getBaiduDownloadBuffer}});var get_buffer_2=require("./ali/get-buffer");Object.defineProperty(exports,"getAliPreviewBuffer",{enumerable:!0,get:function(){return get_buffer_2.getAliPreviewBuffer}}),Object.defineProperty(exports,"getAliDownloadBuffer",{enumerable:!0,get:function(){return get_buffer_2.getAliDownloadBuffer}}),__exportStar(require("./common/types"),exports);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/fetcher",
-  "version": "0.1.23",
+  "version": "0.2.33",
   "publishConfig": {
     "access": "public"
   },
@@ -45,13 +45,13 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@soga/baidu-ua": "^0.1.23",
-    "@soga/entities": "^0.1.23",
-    "@soga/global": "^0.1.23"
+    "@soga/baidu-ua": "^0.2.33",
+    "@soga/bridge": "^0.2.33",
+    "@soga/entities": "^0.2.33"
   },
   "peerDependencies": {
     "axios": "*",
     "typeorm": "*"
   },
-  "gitHead": "4a8d47620964c62bbbcdb10ed7cc2b177c4a10e6"
+  "gitHead": "70d4fae5e473e2f40b7ced6ca434c7b29a527547"
 }

package/dist/ali-buffer.d.ts

@@ -1,2 +0,0 @@
-import { AliBufferParams } from './types';
-export declare function getAliBuffer(params: AliBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/ali-buffer.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliBuffer=getAliBuffer;const ali_fetcher_1=require("./ali-fetcher"),common_1=require("./common");async function getAliBuffer(e){const t=calculateRangeData(e);return await parallelGetBuffer(t)}async function parallelGetBuffer(e){const{ranges:t,threads:a}=e,r=[...t],n=async()=>{if(!r.length)return;const e=r.shift();if(!e)return;const t=(0,common_1.getKey)(e);if(!common_1.bufferCache[t]?.data){const a=(0,ali_fetcher_1.getAliFetcher)(),{access_token:r,drive_id:n,file_id:c}=e,o=await a.getArrayBuffer({access_token:r,drive_id:n,file_id:c,start:e.start,end:e.end,ua:e.ua});common_1.bufferCache[t]={updated:Date.now(),data:Buffer.from(o)}}await n()},c=[];for(let e=0;e<a;e++)c.push(n());await Promise.all(c);const o=t.map((e=>{const t=(0,common_1.getKey)(e);return common_1.bufferCache[t]?.data?common_1.bufferCache[t].data:null})),f=Buffer.concat(o);return t.forEach((e=>{const t=(0,common_1.getKey)(e);delete common_1.bufferCache[t]})),f}function calculateRangeData(e){const{start:t,end:a}=e,r=Math.max(a-t+1,0),n=Math.ceil(r/1572864),c=Math.ceil(r/n),o=[];for(let r=0;r<n;r++){const n=t+r*c,f=Math.min(a,n+c-1);o.push({...e,start:n,end:f})}return{ranges:o,threads:2}}

package/dist/ali-fetcher.js

@@ -1 +0,0 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliFetcher=exports.AliFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),global_1=require("@soga/global"),entities_1=require("@soga/entities"),processMap={};let instance=null;class AliFetcher{axios;domain="https://openapi.alipan.com";urlRepository;constructor(){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e)));const e=(0,global_1.getDataSource)();this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:r}){const a=this.getKey({access_token:e,file_id:i,ua:r});try{for(;processMap[a];)await new Promise((e=>setTimeout(e,100)));processMap[a]=!0;const o=await this.urlRepository.findOne({where:{key:a,fail:(0,typeorm_1.LessThan)(2),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{fail:"ASC"}});if(o)return o.fail>1?(await this.urlRepository.delete({key:a}),""):o.dlink;const{url:s,expiration:n}=await this.fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r});if(s)return await this.urlRepository.save({dlink:s,key:a,speed:100,expire:new Date(n).getTime()}),s;throw new Error("ali fetch download url failed")}finally{delete processMap[a]}}async getArrayBuffer(e){return await this.multipleGetArrayBuffer({...e,times:0})}async multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:o=0,ua:s}){const n=await this.getDownloadUrl({access_token:e,drive_id:t,file_id:i,ua:s});if(!n)throw new Error("获取下载链接失败");try{const e=a-r+1,t=Math.max(1e3*Math.ceil(e/1024/50),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${r}-${a}`,"User-Agent":s,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(l){const c=this.getKey({access_token:e,file_id:i,ua:s});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(o||0)<3)return await this.multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:i,start:r,end:a,times:(o||0)+1,ua:s});throw l}}async fetchDownloadInfo({access_token:e,drive_id:t,file_id:i,ua:r}){const a={Authorization:`Bearer ${e}`,"User-Agent":r},{url:o,expiration:s}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:t,file_id:i},{headers:a});return{url:o,expiration:s}}getKey({access_token:e,file_id:t,ua:i}){const r=i?encodeURIComponent(i):"ua";return`ali-${encodeURIComponent(e)}-${r}-${t}`}}exports.AliFetcher=AliFetcher;const getAliFetcher=()=>(instance||((0,global_1.checkDataSource)(),instance=new AliFetcher),instance);exports.getAliFetcher=getAliFetcher;

package/dist/baidu-buffer.d.ts

@@ -1,2 +0,0 @@
-import { BaiduBufferParams } from './types';
-export declare function getBaiduBuffer(params: BaiduBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/baidu-buffer.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduBuffer=getBaiduBuffer;const baidu_ua_1=require("@soga/baidu-ua"),baidu_fetcher_1=require("./baidu-fetcher"),common_1=require("./common");async function getBaiduBuffer(t){const e=calculateRangeData(t);return await parallelGetBuffer(e)}async function parallelGetBuffer(t){const{ranges:e,threads:a}=t,r=[...e],n=async()=>{if(!r.length)return;const t=r.shift();if(!t)return;const e=(0,common_1.getKey)(t);if(!common_1.bufferCache[e]?.data){const a=(0,baidu_fetcher_1.getBaiduFetcher)(),{access_token:r,fs_id:n,host_id:u}=t,{ua:o,finishBaiduUA:c}=await(0,baidu_ua_1.getBaiduUA)(u,t.total),s=await a.getArrayBuffer({access_token:r,fs_id:n,start:t.start,end:t.end,ua:t.auto_ua?o:t.ua});c(),common_1.bufferCache[e]={updated:Date.now(),data:Buffer.from(s)}}await n()},u=[];for(let t=0;t<a;t++)u.push(n());await Promise.all(u);const o=e.map((t=>{const e=(0,common_1.getKey)(t);return common_1.bufferCache[e]?.data?common_1.bufferCache[e].data:null})),c=Buffer.concat(o);return e.forEach((t=>{const e=(0,common_1.getKey)(t);delete common_1.bufferCache[e]})),c}function calculateRangeData(t){const{start:e,end:a,total:r}=t,n=Math.max(a-e+1,0);if(2==t.host_vip_type){const r=Math.ceil(n/1572864),u=Math.ceil(n/r),o=[];for(let n=0;n<r;n++){const r=e+n*u,c=Math.min(a,r+u-1);o.push({...t,start:r,end:c,auto_ua:!1})}return{ranges:o,threads:2}}{const u=Math.ceil(n/131072),o=Math.ceil(n/u),c=[];for(let r=0;r<u;r++){const n=e+r*o,u=Math.min(a,n+o-1);c.push({...t,start:n,end:u,auto_ua:!0})}return{ranges:c,threads:Math.min(r<52428800?8:6,u)}}}

package/dist/baidu-fetcher.js

@@ -1 +0,0 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduFetcher=exports.BaiduFetcher=void 0;const axios_1=__importDefault(require("axios")),typeorm_1=require("typeorm"),global_1=require("@soga/global"),entities_1=require("@soga/entities"),processMap={};let instance=null,lastCleanTime=0;class BaiduFetcher{axios;domain="https://pan.baidu.com";urlRepository;constructor(){this.axios=axios_1.default.create({baseURL:this.domain}),this.axios.interceptors.response.use((e=>e.data),(e=>Promise.reject(e)));const e=(0,global_1.getDataSource)();this.urlRepository=e.getRepository(entities_1.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:s=0,ua:i}){const r=this.getKey({fs_id:t,access_token:e,ua:i});try{for(;processMap[r];)await new Promise((e=>setTimeout(e,100)));processMap[r]=!0;const n=await this.urlRepository.findOne({where:{key:r,speed:(0,typeorm_1.MoreThan)(30),expire:(0,typeorm_1.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:r}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:s,ua:i});if(o)return await this.urlRepository.save({dlink:o,key:r,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete processMap[r]}}async cleanExpired(){Date.now()-lastCleanTime>18e5&&(await this.urlRepository.delete({expire:(0,typeorm_1.LessThan)(Date.now())}),lastCleanTime=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:s,access_token:i}){const r=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:i,ua:s});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":s,Range:`bytes=${t}-${c}`},l=c-t+1,u={start:0,end:0};for(let e=0;e<50;e++){const e=(await axios_1.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!r.includes(a)){n=e;try{return u.start=Date.now(),await axios_1.default.get(e,{headers:d,timeout:2e3}),u.end=Date.now(),{dlink:e,speed:1e3*l/(u.end-u.start)}}catch(e){r.push(a),await new Promise((e=>setTimeout(e,200)));continue}}}return{dlink:n,speed:1e3*l/(u.end-u.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const s=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:i,list:r}=s||{};if(0!=i)throw new Error(`获取下载链接失败: ${i}`);if(!r||!r.length)throw new Error("获取下载链接失败");return`${r[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:i=0,ua:r}){const n=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:s,ua:r});if(!n)throw new Error("获取下载链接失败");try{const e=s-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await axios_1.default.get(n,{headers:{Range:`bytes=${a}-${s}`,"User-Agent":r,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t})).data}catch(o){const c=this.getKey({fs_id:t,access_token:e,ua:r});if(await this.urlRepository.increment({key:c,dlink:n},"fail",1),await new Promise((e=>setTimeout(e,1e3))),(i||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:s,times:(i||0)+1,ua:r});throw o}}getKey({access_token:e,fs_id:t,ua:a}){const s=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${s}-${t}`}}exports.BaiduFetcher=BaiduFetcher;const getBaiduFetcher=()=>(instance||((0,global_1.checkDataSource)(),instance=new BaiduFetcher),instance);exports.getBaiduFetcher=getBaiduFetcher;

package/dist/preview-buffer.d.ts

@@ -1,3 +0,0 @@
-import { AliPreviewBufferParams, BaiduPreviewBufferParams } from './types';
-export declare const getBaiduPreviewBuffer: ({ record_id, start, end, total, part_name, access_token, host_id, host_vip_type, fs_id, cache, cache_folder, }: BaiduPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;
-export declare const getAliPreviewBuffer: ({ record_id, start, end, total, part_name, access_token, host_id, host_vip_type, drive_id, file_id, cache, cache_folder, }: AliPreviewBufferParams) => Promise<Buffer<ArrayBufferLike>>;

package/dist/preview-buffer.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliPreviewBuffer=exports.getBaiduPreviewBuffer=void 0;const ali_buffer_1=require("./ali-buffer"),baidu_buffer_1=require("./baidu-buffer"),preview_store_1=require("./stores/preview-store"),getKey=e=>`${e.part_name}_${e.start}_${e.end}`,getBaiduPreviewBuffer=async({record_id:e,start:t,end:r,total:i,part_name:a,access_token:_,host_id:f,host_vip_type:s,fs_id:o,cache:d,cache_folder:u})=>{const n=(0,preview_store_1.getPreviewStore)(u),c=getKey({part_name:a,start:t,end:r}),p=await n.readBuffer(c);if(p)return p;const v=await(0,baidu_buffer_1.getBaiduBuffer)({record_id:e,start:t,end:r,total:i,access_token:_,host_id:f,host_vip_type:s,fs_id:o,part_name:a});return d&&await n.writeBuffer(c,v),v};exports.getBaiduPreviewBuffer=getBaiduPreviewBuffer;const getAliPreviewBuffer=async({record_id:e,start:t,end:r,total:i,part_name:a,access_token:_,host_id:f,host_vip_type:s,drive_id:o,file_id:d,cache:u,cache_folder:n})=>{const c=(0,preview_store_1.getPreviewStore)(n),p=getKey({part_name:a,start:t,end:r}),v=await c.readBuffer(p);if(v)return v;const w=await(0,ali_buffer_1.getAliBuffer)({record_id:e,start:t,end:r,total:i,access_token:_,host_id:f,host_vip_type:s,drive_id:o,file_id:d,part_name:a});return u&&await c.writeBuffer(p,w),w};exports.getAliPreviewBuffer=getAliPreviewBuffer;

package/dist/stores/preview-store.d.ts

@@ -1,15 +0,0 @@
-import { Level } from 'level';
-import { AbstractSublevel } from 'abstract-level';
-export declare class PreviewStore {
-    folder_path: string;
-    db: Level<string, any>;
-    sub: AbstractSublevel<Level<string, any>, string | Buffer<ArrayBufferLike> | Uint8Array<ArrayBufferLike>, string, Buffer<ArrayBuffer | ArrayBufferLike>>;
-    constructor(folder_path: string);
-    read<V>(key: string): Promise<V>;
-    write<V>(key: string, value: V): Promise<void>;
-    delete(key: string): Promise<void>;
-    writeBuffer(key: string, buffer: Buffer<ArrayBuffer | ArrayBufferLike>): Promise<void>;
-    readBuffer(key: string): Promise<Buffer<ArrayBufferLike>>;
-    deleteBuffer(key: string): Promise<void>;
-}
-export declare const getPreviewStore: (folder_path: string) => PreviewStore;

package/dist/stores/preview-store.js

@@ -1 +0,0 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getPreviewStore=exports.PreviewStore=void 0;const level_1=require("level"),path_1=require("path");class PreviewStore{folder_path;db;sub;constructor(e){this.folder_path=e,this.db=new level_1.Level((0,path_1.resolve)(e,"preview_cache"),{valueEncoding:"json",maxFileSize:67108864}),this.sub=this.db.sublevel("sub",{valueEncoding:"buffer"})}async read(e){return await this.db.get(e,{})}async write(e,t){await this.db.put(e,t,{})}async delete(e){await this.db.del(e)}async writeBuffer(e,t){await this.sub.put(e,t)}async readBuffer(e){return await this.sub.get(e)}async deleteBuffer(e){await this.sub.del(e)}}exports.PreviewStore=PreviewStore;const storeMap={},getPreviewStore=e=>{if(storeMap[e])return storeMap[e].updated_at=Date.now(),storeMap[e].store;const t=new PreviewStore(e);return storeMap[e]={updated_at:Date.now(),store:t},t};exports.getPreviewStore=getPreviewStore;