@soga/fetcher 0.3.0 → 1.3.1

package/dist/main.d.mts

@@ -0,0 +1,103 @@
+import { AxiosInstance } from 'axios';
+import { Repository, DataSource } from 'typeorm';
+import { FetchUrl } from '@soga/entities';
+
+declare class BaiduFetcher {
+    axios: AxiosInstance;
+    private domain;
+    protected urlRepository: Repository<FetchUrl>;
+    constructor(dataSource: DataSource);
+    getDownloadUrl({ access_token, fs_id, ua, }: {
+        access_token: string;
+        fs_id: number;
+        ua?: string;
+    }): Promise<string>;
+    getArrayBuffer(params: {
+        access_token: string;
+        fs_id: number;
+        start: number;
+        end: number;
+        abort_controller?: AbortController;
+        ua?: string;
+    }): Promise<ArrayBuffer>;
+    getDownloadUrlByRange({ access_token, fs_id, start, end, ua, }: {
+        access_token: string;
+        fs_id: number;
+        start: number;
+        end: number;
+        ua?: string;
+    }): Promise<string>;
+    private cleanExpired;
+    private fetchBestDlink;
+    private fetchDownloadUrl;
+    private multipleGetArrayBuffer;
+    private getKey;
+}
+declare const getBaiduFetcher: (dataSource: DataSource) => BaiduFetcher;
+
+declare class AliFetcher {
+    axios: AxiosInstance;
+    private domain;
+    protected urlRepository: Repository<FetchUrl>;
+    constructor(dataSource: DataSource);
+    getDownloadUrl({ access_token, drive_id, file_id, ua, }: {
+        access_token: string;
+        drive_id: string;
+        file_id: string;
+        ua?: string;
+    }): Promise<string>;
+    getArrayBuffer(params: {
+        access_token: string;
+        drive_id: string;
+        file_id: string;
+        start: number;
+        end: number;
+        ua?: string;
+        abort_controller?: AbortController;
+    }): Promise<ArrayBuffer>;
+    private multipleGetArrayBuffer;
+    private fetchDownloadInfo;
+    private getKey;
+}
+declare const getAliFetcher: (dataSource: DataSource) => AliFetcher;
+
+interface CommonBufferParams {
+    id?: string;
+    dataSource: DataSource;
+    record_id: number;
+    part_md5: string;
+    start: number;
+    end: number;
+    total: number;
+    host_id: number;
+    host_vip_type: number;
+    access_token: string;
+    abort_controller?: AbortController;
+    ua?: string;
+}
+interface BaiduBufferParams extends CommonBufferParams {
+    fs_id: number;
+}
+interface AliBufferParams extends CommonBufferParams {
+    drive_id: string;
+    file_id: string;
+}
+interface BaiduPreviewBufferParams extends BaiduBufferParams {
+    cache_folder: string;
+}
+interface AliPreviewBufferParams extends AliBufferParams {
+    cache_folder: string;
+}
+type CacheType = 'preview' | 'download';
+
+declare function fetchBaiduBuffer(params: BaiduBufferParams): Promise<Buffer<ArrayBuffer>>;
+
+declare function fetchAliBuffer(params: AliBufferParams): Promise<Buffer<ArrayBuffer>>;
+
+declare const getBaiduPreviewBuffer: (params: BaiduPreviewBufferParams) => Promise<any>;
+declare const getBaiduDownloadBuffer: (params: BaiduPreviewBufferParams) => Promise<any>;
+
+declare const getAliPreviewBuffer: (params: AliPreviewBufferParams) => Promise<any>;
+declare const getAliDownloadBuffer: (params: AliPreviewBufferParams) => Promise<any>;
+
+export { type AliBufferParams, AliFetcher, type AliPreviewBufferParams, type BaiduBufferParams, BaiduFetcher, type BaiduPreviewBufferParams, type CacheType, fetchAliBuffer, fetchBaiduBuffer, getAliDownloadBuffer, getAliFetcher, getAliPreviewBuffer, getBaiduDownloadBuffer, getBaiduFetcher, getBaiduPreviewBuffer };

package/dist/main.d.ts

@@ -1,7 +1,103 @@
-export { BaiduFetcher, getBaiduFetcher } from './baidu/fetcher';
-export { AliFetcher, getAliFetcher } from './ali/fetcher';
-export { fetchBaiduBuffer } from './baidu/fetch-buffer';
-export { fetchAliBuffer } from './ali/fetch-buffer';
-export { getBaiduPreviewBuffer, getBaiduDownloadBuffer, } from './baidu/get-buffer';
-export { getAliPreviewBuffer, getAliDownloadBuffer } from './ali/get-buffer';
-export * from './common/types';
+import { AxiosInstance } from 'axios';
+import { Repository, DataSource } from 'typeorm';
+import { FetchUrl } from '@soga/entities';
+
+declare class BaiduFetcher {
+    axios: AxiosInstance;
+    private domain;
+    protected urlRepository: Repository<FetchUrl>;
+    constructor(dataSource: DataSource);
+    getDownloadUrl({ access_token, fs_id, ua, }: {
+        access_token: string;
+        fs_id: number;
+        ua?: string;
+    }): Promise<string>;
+    getArrayBuffer(params: {
+        access_token: string;
+        fs_id: number;
+        start: number;
+        end: number;
+        abort_controller?: AbortController;
+        ua?: string;
+    }): Promise<ArrayBuffer>;
+    getDownloadUrlByRange({ access_token, fs_id, start, end, ua, }: {
+        access_token: string;
+        fs_id: number;
+        start: number;
+        end: number;
+        ua?: string;
+    }): Promise<string>;
+    private cleanExpired;
+    private fetchBestDlink;
+    private fetchDownloadUrl;
+    private multipleGetArrayBuffer;
+    private getKey;
+}
+declare const getBaiduFetcher: (dataSource: DataSource) => BaiduFetcher;
+
+declare class AliFetcher {
+    axios: AxiosInstance;
+    private domain;
+    protected urlRepository: Repository<FetchUrl>;
+    constructor(dataSource: DataSource);
+    getDownloadUrl({ access_token, drive_id, file_id, ua, }: {
+        access_token: string;
+        drive_id: string;
+        file_id: string;
+        ua?: string;
+    }): Promise<string>;
+    getArrayBuffer(params: {
+        access_token: string;
+        drive_id: string;
+        file_id: string;
+        start: number;
+        end: number;
+        ua?: string;
+        abort_controller?: AbortController;
+    }): Promise<ArrayBuffer>;
+    private multipleGetArrayBuffer;
+    private fetchDownloadInfo;
+    private getKey;
+}
+declare const getAliFetcher: (dataSource: DataSource) => AliFetcher;
+
+interface CommonBufferParams {
+    id?: string;
+    dataSource: DataSource;
+    record_id: number;
+    part_md5: string;
+    start: number;
+    end: number;
+    total: number;
+    host_id: number;
+    host_vip_type: number;
+    access_token: string;
+    abort_controller?: AbortController;
+    ua?: string;
+}
+interface BaiduBufferParams extends CommonBufferParams {
+    fs_id: number;
+}
+interface AliBufferParams extends CommonBufferParams {
+    drive_id: string;
+    file_id: string;
+}
+interface BaiduPreviewBufferParams extends BaiduBufferParams {
+    cache_folder: string;
+}
+interface AliPreviewBufferParams extends AliBufferParams {
+    cache_folder: string;
+}
+type CacheType = 'preview' | 'download';
+
+declare function fetchBaiduBuffer(params: BaiduBufferParams): Promise<Buffer<ArrayBuffer>>;
+
+declare function fetchAliBuffer(params: AliBufferParams): Promise<Buffer<ArrayBuffer>>;
+
+declare const getBaiduPreviewBuffer: (params: BaiduPreviewBufferParams) => Promise<any>;
+declare const getBaiduDownloadBuffer: (params: BaiduPreviewBufferParams) => Promise<any>;
+
+declare const getAliPreviewBuffer: (params: AliPreviewBufferParams) => Promise<any>;
+declare const getAliDownloadBuffer: (params: AliPreviewBufferParams) => Promise<any>;
+
+export { type AliBufferParams, AliFetcher, type AliPreviewBufferParams, type BaiduBufferParams, BaiduFetcher, type BaiduPreviewBufferParams, type CacheType, fetchAliBuffer, fetchBaiduBuffer, getAliDownloadBuffer, getAliFetcher, getAliPreviewBuffer, getBaiduDownloadBuffer, getBaiduFetcher, getBaiduPreviewBuffer };

package/dist/main.js

@@ -1,34 +1 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAliDownloadBuffer = exports.getAliPreviewBuffer = exports.getBaiduDownloadBuffer = exports.getBaiduPreviewBuffer = exports.fetchAliBuffer = exports.fetchBaiduBuffer = exports.getAliFetcher = exports.AliFetcher = exports.getBaiduFetcher = exports.BaiduFetcher = void 0;
-var fetcher_1 = require("./baidu/fetcher");
-Object.defineProperty(exports, "BaiduFetcher", { enumerable: true, get: function () { return fetcher_1.BaiduFetcher; } });
-Object.defineProperty(exports, "getBaiduFetcher", { enumerable: true, get: function () { return fetcher_1.getBaiduFetcher; } });
-var fetcher_2 = require("./ali/fetcher");
-Object.defineProperty(exports, "AliFetcher", { enumerable: true, get: function () { return fetcher_2.AliFetcher; } });
-Object.defineProperty(exports, "getAliFetcher", { enumerable: true, get: function () { return fetcher_2.getAliFetcher; } });
-var fetch_buffer_1 = require("./baidu/fetch-buffer");
-Object.defineProperty(exports, "fetchBaiduBuffer", { enumerable: true, get: function () { return fetch_buffer_1.fetchBaiduBuffer; } });
-var fetch_buffer_2 = require("./ali/fetch-buffer");
-Object.defineProperty(exports, "fetchAliBuffer", { enumerable: true, get: function () { return fetch_buffer_2.fetchAliBuffer; } });
-var get_buffer_1 = require("./baidu/get-buffer");
-Object.defineProperty(exports, "getBaiduPreviewBuffer", { enumerable: true, get: function () { return get_buffer_1.getBaiduPreviewBuffer; } });
-Object.defineProperty(exports, "getBaiduDownloadBuffer", { enumerable: true, get: function () { return get_buffer_1.getBaiduDownloadBuffer; } });
-var get_buffer_2 = require("./ali/get-buffer");
-Object.defineProperty(exports, "getAliPreviewBuffer", { enumerable: true, get: function () { return get_buffer_2.getAliPreviewBuffer; } });
-Object.defineProperty(exports, "getAliDownloadBuffer", { enumerable: true, get: function () { return get_buffer_2.getAliDownloadBuffer; } });
-__exportStar(require("./common/types"), exports);
+var e,t=Object.create,a=Object.defineProperty,r=Object.getOwnPropertyDescriptor,i=Object.getOwnPropertyNames,s=Object.getPrototypeOf,n=Object.prototype.hasOwnProperty,o=(e,t,s,o)=>{if(t&&"object"==typeof t||"function"==typeof t)for(let c of i(t))n.call(e,c)||c===s||a(e,c,{get:()=>t[c],enumerable:!(o=r(t,c))||o.enumerable});return e},c=(e,r,i)=>(i=null!=e?t(s(e)):{},o(!r&&e&&e.__esModule?i:a(i,"default",{value:e,enumerable:!0}),e)),d={};((e,t)=>{for(var r in t)a(e,r,{get:t[r],enumerable:!0})})(d,{AliFetcher:()=>v,BaiduFetcher:()=>p,fetchAliBuffer:()=>M,fetchBaiduBuffer:()=>U,getAliDownloadBuffer:()=>K,getAliFetcher:()=>D,getAliPreviewBuffer:()=>I,getBaiduDownloadBuffer:()=>C,getBaiduFetcher:()=>y,getBaiduPreviewBuffer:()=>F}),module.exports=(e=d,o(a({},"__esModule",{value:!0}),e));var l=c(require("axios")),u=require("typeorm"),f=require("@soga/entities"),h={},w=null,_=0,p=class{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=l.default.create({baseURL:this.domain}),this.axios.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),this.urlRepository=e.getRepository(f.FetchUrl)}async getDownloadUrl({access_token:e,fs_id:t,ua:a}){return await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:0,end:0,ua:a})}async getArrayBuffer(e){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...e,times:0})}async getDownloadUrlByRange({access_token:e,fs_id:t,start:a=0,end:r=0,ua:i}){const s=this.getKey({fs_id:t,access_token:e,ua:i});try{for(;h[s];)await new Promise(e=>setTimeout(e,100));h[s]=!0;const n=await this.urlRepository.findOne({where:{key:s,speed:(0,u.MoreThan)(30),expire:(0,u.MoreThan)(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(n)return n.fail>10?(await this.urlRepository.delete({key:s}),""):n.dlink;const{dlink:o,speed:c}=await this.fetchBestDlink({access_token:e,fs_id:t,start:a,end:r,ua:i});if(o)return await this.urlRepository.save({dlink:o,key:s,speed:c,expire:Date.now()+252e5}),o;throw new Error("baidu fetch download url failed")}finally{delete h[s]}}async cleanExpired(){Date.now()-_>18e5&&(await this.urlRepository.delete({expire:(0,u.LessThan)(Date.now())}),_=Date.now())}async fetchBestDlink({fs_id:e,start:t=0,end:a=0,ua:r,access_token:i}){const s=[];let n="";const o=await this.fetchDownloadUrl({fs_id:e,access_token:i,ua:r});if(!o)return{dlink:"",speed:0};const c=Math.min(t+127,a),d={"User-Agent":r,Range:`bytes=${t}-${c}`},u=c-t+1,f={start:0,end:0};for(let e=0;e<50;e++){const e=(await l.default.get(o,{headers:d,maxRedirects:0,validateStatus:e=>e>=200&&e<303})).headers.location;if(!e)return{dlink:"",speed:0};const t=new URL(e),{hostname:a}=t;if(!s.includes(a)){n=e;try{return f.start=Date.now(),await l.default.get(e,{headers:d,timeout:2e3}),f.end=Date.now(),{dlink:e,speed:1e3*u/(f.end-f.start)}}catch(e){s.push(a),await new Promise(e=>setTimeout(e,200));continue}}}return{dlink:n,speed:1e3*u/(f.end-f.start)}}async fetchDownloadUrl({fs_id:e,ua:t="pan.baidu.com",access_token:a}){const r=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([e])},headers:{"User-Agent":t}}),{errno:i,list:s}=r||{};if(0!=i)throw new Error(`获取下载链接失败: ${i}`);if(!s||!s.length)throw new Error("获取下载链接失败");return`${s[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:r,abort_controller:i,times:s=0,ua:n}){const o=await this.getDownloadUrlByRange({access_token:e,fs_id:t,start:a,end:r,ua:n});if(!o)throw new Error("获取下载链接失败");try{const e=r-a+1,t=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await l.default.get(o,{headers:{Range:`bytes=${a}-${r}`,"User-Agent":n,Accept:"application/octet-stream"},signal:i?.signal,responseType:"arraybuffer",timeout:t})).data}catch(c){if("AbortError"===c.name)return;const d=this.getKey({fs_id:t,access_token:e,ua:n});if(await this.urlRepository.increment({key:d,dlink:o},"fail",1),await new Promise(e=>setTimeout(e,1e3)),(s||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:t,start:a,end:r,times:(s||0)+1,abort_controller:i,ua:n});throw c}}getKey({access_token:e,fs_id:t,ua:a}){const r=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(e)}-${r}-${t}`}},y=e=>(w||(w=new p(e)),w),m=c(require("axios")),g=require("typeorm"),k=require("@soga/entities"),b={},B=null,v=class{axios;domain="https://openapi.alipan.com";urlRepository;constructor(e){this.axios=m.default.create({baseURL:this.domain}),this.axios.interceptors.response.use(e=>e.data,e=>Promise.reject(e)),this.urlRepository=e.getRepository(k.FetchUrl)}async getDownloadUrl({access_token:e,drive_id:t,file_id:a,ua:r}){const i=this.getKey({access_token:e,file_id:a,ua:r});try{for(;b[i];)await new Promise(e=>setTimeout(e,100));b[i]=!0;const s=await this.urlRepository.findOne({where:{key:i,fail:(0,g.LessThan)(2),expire:(0,g.MoreThan)(Date.now())},order:{fail:"ASC"}});if(s)return s.fail>1?(await this.urlRepository.delete({key:i}),""):s.dlink;const{url:n,expiration:o}=await this.fetchDownloadInfo({access_token:e,drive_id:t,file_id:a,ua:r});if(n)return await this.urlRepository.save({dlink:n,key:i,speed:100,expire:new Date(o).getTime()}),n;throw new Error("ali fetch download url failed")}finally{delete b[i]}}async getArrayBuffer(e){return await this.multipleGetArrayBuffer({...e,times:0})}async multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:a,start:r,end:i,abort_controller:s,times:n=0,ua:o}){const c=await this.getDownloadUrl({access_token:e,drive_id:t,file_id:a,ua:o});if(!c)throw new Error("获取下载链接失败");try{const e=i-r+1,t=Math.max(1e3*Math.ceil(e/1024/50),3e3);return(await m.default.get(c,{headers:{Range:`bytes=${r}-${i}`,"User-Agent":o,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:t,signal:s?.signal})).data}catch(s){if("AbortError"===s.name)return;const d=this.getKey({access_token:e,file_id:a,ua:o});if(await this.urlRepository.increment({key:d,dlink:c},"fail",1),await new Promise(e=>setTimeout(e,1e3)),(n||0)<3)return await this.multipleGetArrayBuffer({access_token:e,drive_id:t,file_id:a,start:r,end:i,times:(n||0)+1,ua:o});throw s}}async fetchDownloadInfo({access_token:e,drive_id:t,file_id:a,ua:r}){const i={Authorization:`Bearer ${e}`,"User-Agent":r},{url:s,expiration:n}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:t,file_id:a},{headers:i});return{url:s,expiration:n}}getKey({access_token:e,file_id:t,ua:a}){const r=a?encodeURIComponent(a):"ua";return`ali-${encodeURIComponent(e)}-${r}-${t}`}},D=e=>(B||(B=new v(e)),B),R=require("@soga/baidu-ua"),x={},A=({part_md5:e,start:t,end:a})=>`${e}-${t}-${a}`;async function U(e){const t=function(e){const{start:t,end:a,total:r}=e,i=Math.max(a-t+1,0);if(2==e.host_vip_type){const r=Math.ceil(i/1572864),s=Math.ceil(i/r),n=[];for(let i=0;i<r;i++){const r=t+i*s,o=Math.min(a,r+s-1);n.push({...e,start:r,end:o,auto_ua:!1})}return{ranges:n,threads:2}}{const s=Math.ceil(i/131072),n=Math.ceil(i/s),o=[];for(let r=0;r<s;r++){const i=t+r*n,s=Math.min(a,i+n-1);o.push({...e,start:i,end:s,auto_ua:!0})}return{ranges:o,threads:Math.min(r<52428800?8:6,s)}}}(e),a=await async function(e,t){const{ranges:a,threads:r}=e,i=[...a],s=async()=>{if(!i.length)return;const e=i.shift();if(!e)return;const a=A(e);if(!x[a]?.data){const r=y(t),{access_token:i,fs_id:s,host_id:n}=e,{ua:o,finish:c}=await(0,R.getDownloadBaiduUA)(n,e.total),d=await r.getArrayBuffer({access_token:i,fs_id:s,start:e.start,end:e.end,ua:e.auto_ua?o:e.ua});if(c(),!d||0===d.byteLength)return;x[a]={updated:Date.now(),data:Buffer.from(d)}}await s()},n=[];for(let e=0;e<r;e++)n.push(s());await Promise.all(n);let o=!0;const c=a.map(e=>{const t=A(e);return x[t]?.data?x[t].data:(o=!1,null)});if(o){const e=Buffer.concat(c);return a.forEach(e=>{const t=A(e);delete x[t]}),e}return null}(t,e.dataSource);return a}async function M(e){const t=function(e){const{start:t,end:a}=e,r=Math.max(a-t+1,0),i=Math.ceil(r/1572864),s=Math.ceil(r/i),n=[];for(let r=0;r<i;r++){const i=t+r*s,o=Math.min(a,i+s-1);n.push({...e,start:i,end:o})}return{ranges:n,threads:2}}(e),a=await async function(e,t){const{ranges:a,threads:r}=e,i=[...a],s=async()=>{if(!i.length)return;const e=i.shift();if(!e)return;const a=A(e);if(!x[a]?.data){const r=D(t),{access_token:i,drive_id:s,file_id:n}=e,o=await r.getArrayBuffer({access_token:i,drive_id:s,file_id:n,start:e.start,end:e.end,ua:e.ua,abort_controller:e.abort_controller});x[a]={updated:Date.now(),data:Buffer.from(o)}}await s()},n=[];for(let e=0;e<r;e++)n.push(s());await Promise.all(n);let o=!0;const c=a.map(e=>{const t=A(e);return x[t]?.data?x[t].data:(o=!1,null)});if(o){const e=Buffer.concat(c);return a.forEach(e=>{const t=A(e);delete x[t]}),e}return null}(t,e.dataSource);return a}var $=require("@soga/lowdb"),P=require("fs-extra"),E=require("path"),O=class{folder_path;cache_path;db;cache_type;constructor(e,t){this.folder_path=e,this.cache_type=t,this.cache_path=(0,E.resolve)(e,`${t}_cache`)}async init(){const e=(0,E.resolve)(this.cache_path,"cache.json");this.db=await(0,$.getDb)(e,{})}async read(e){return this.db.data[e]}async write(e,t){this.db.data[e]=t,await this.db.write()}async delete(e){delete this.db.data[e],await this.db.write()}async writeBuffer(e,t){const a=(0,E.resolve)(this.cache_path,e);await(0,P.writeFile)(a,t),this.db.data[e]=!0,await this.db.write()}async readBuffer(e){if(!this.db.data[e])return null;const t=(0,E.resolve)(this.cache_path,e);return await(0,P.readFile)(t)}async deleteBuffer(e){if(!this.db.data[e])return;delete this.db.data[e],await this.db.write();const t=(0,E.resolve)(this.cache_path,e);try{await(0,P.remove)(t)}catch(e){}}},T={},j=0,S=async(e,t)=>{if(T[e]){for(;!T[e].inited;)await new Promise(e=>setTimeout(e,20));T[e].updated_at=Date.now();const t=Date.now();return t-j>18e5&&(j=t,Object.keys(T).forEach(e=>{t-T[e].updated_at>288e5&&delete T[e]})),T[e].store}const a=new O(e,t);return T[e]={updated_at:Date.now(),store:a,inited:!1},await a.init(),T[e].inited=!0,a},q=async({id:e,dataSource:t,record_id:a,start:r,end:i,total:s,part_md5:n,access_token:o,host_id:c,host_vip_type:d,fs_id:l,cache_folder:u,abort_controller:f},h)=>{let w=null;const _=e||A({part_md5:n,start:r,end:i});if(u){w=await S(u,h);const e=await w.readBuffer(_);if(e)return e}const p=await U({dataSource:t,record_id:a,start:r,end:i,total:s,access_token:o,host_id:c,host_vip_type:d,fs_id:l,part_md5:n,abort_controller:f});return p&&w&&await w.writeBuffer(_,p),p},F=async e=>await q(e,"preview"),C=async e=>await q(e,"download"),G=async({id:e,dataSource:t,record_id:a,start:r,end:i,total:s,part_md5:n,access_token:o,host_id:c,host_vip_type:d,drive_id:l,file_id:u,cache_folder:f,abort_controller:h},w)=>{const _=e||A({part_md5:n,start:r,end:i});let p=null;if(f){p=await S(f,w);const e=await p.readBuffer(_);if(e)return e}const y=await M({dataSource:t,record_id:a,start:r,end:i,total:s,access_token:o,host_id:c,host_vip_type:d,drive_id:l,file_id:u,part_md5:n,abort_controller:h});return y&&p&&await p.writeBuffer(_,y),y},I=async e=>await G(e,"preview"),K=async e=>await G(e,"download");

package/dist/main.mjs

@@ -0,0 +1 @@
+import t from"axios";import{LessThan as e,MoreThan as a}from"typeorm";import{FetchUrl as r}from"@soga/entities";var i={},s=null,n=0,o=class{axios;domain="https://pan.baidu.com";urlRepository;constructor(e){this.axios=t.create({baseURL:this.domain}),this.axios.interceptors.response.use(t=>t.data,t=>Promise.reject(t)),this.urlRepository=e.getRepository(r)}async getDownloadUrl({access_token:t,fs_id:e,ua:a}){return await this.getDownloadUrlByRange({access_token:t,fs_id:e,start:0,end:0,ua:a})}async getArrayBuffer(t){return await this.cleanExpired(),await this.multipleGetArrayBuffer({...t,times:0})}async getDownloadUrlByRange({access_token:t,fs_id:e,start:r=0,end:s=0,ua:n}){const o=this.getKey({fs_id:e,access_token:t,ua:n});try{for(;i[o];)await new Promise(t=>setTimeout(t,100));i[o]=!0;const c=await this.urlRepository.findOne({where:{key:o,speed:a(30),expire:a(Date.now())},order:{speed:"DESC",fail:"ASC"}});if(c)return c.fail>10?(await this.urlRepository.delete({key:o}),""):c.dlink;const{dlink:d,speed:l}=await this.fetchBestDlink({access_token:t,fs_id:e,start:r,end:s,ua:n});if(d)return await this.urlRepository.save({dlink:d,key:o,speed:l,expire:Date.now()+252e5}),d;throw new Error("baidu fetch download url failed")}finally{delete i[o]}}async cleanExpired(){Date.now()-n>18e5&&(await this.urlRepository.delete({expire:e(Date.now())}),n=Date.now())}async fetchBestDlink({fs_id:e,start:a=0,end:r=0,ua:i,access_token:s}){const n=[];let o="";const c=await this.fetchDownloadUrl({fs_id:e,access_token:s,ua:i});if(!c)return{dlink:"",speed:0};const d=Math.min(a+127,r),l={"User-Agent":i,Range:`bytes=${a}-${d}`},u=d-a+1,f={start:0,end:0};for(let e=0;e<50;e++){const e=(await t.get(c,{headers:l,maxRedirects:0,validateStatus:t=>t>=200&&t<303})).headers.location;if(!e)return{dlink:"",speed:0};const a=new URL(e),{hostname:r}=a;if(!n.includes(r)){o=e;try{return f.start=Date.now(),await t.get(e,{headers:l,timeout:2e3}),f.end=Date.now(),{dlink:e,speed:1e3*u/(f.end-f.start)}}catch(t){n.push(r),await new Promise(t=>setTimeout(t,200));continue}}}return{dlink:o,speed:1e3*u/(f.end-f.start)}}async fetchDownloadUrl({fs_id:t,ua:e="pan.baidu.com",access_token:a}){const r=await this.axios.get("/rest/2.0/xpan/multimedia",{params:{access_token:a,dlink:"1",method:"filemetas",fsids:JSON.stringify([t])},headers:{"User-Agent":e}}),{errno:i,list:s}=r||{};if(0!=i)throw new Error(`获取下载链接失败: ${i}`);if(!s||!s.length)throw new Error("获取下载链接失败");return`${s[0].dlink}&access_token=${a}`}async multipleGetArrayBuffer({access_token:e,fs_id:a,start:r,end:i,abort_controller:s,times:n=0,ua:o}){const c=await this.getDownloadUrlByRange({access_token:e,fs_id:a,start:r,end:i,ua:o});if(!c)throw new Error("获取下载链接失败");try{const e=i-r+1,a=Math.max(1e3*Math.ceil(e/1024/10),3e3);return(await t.get(c,{headers:{Range:`bytes=${r}-${i}`,"User-Agent":o,Accept:"application/octet-stream"},signal:s?.signal,responseType:"arraybuffer",timeout:a})).data}catch(t){if("AbortError"===t.name)return;const d=this.getKey({fs_id:a,access_token:e,ua:o});if(await this.urlRepository.increment({key:d,dlink:c},"fail",1),await new Promise(t=>setTimeout(t,1e3)),(n||0)<3)return await this.multipleGetArrayBuffer({access_token:e,fs_id:a,start:r,end:i,times:(n||0)+1,abort_controller:s,ua:o});throw t}}getKey({access_token:t,fs_id:e,ua:a}){const r=a?encodeURIComponent(a):"ua";return`baidu-${encodeURIComponent(t)}-${r}-${e}`}},c=t=>(s||(s=new o(t)),s);import d from"axios";import{LessThan as l,MoreThan as u}from"typeorm";import{FetchUrl as f}from"@soga/entities";var h={},w=null,_=class{axios;domain="https://openapi.alipan.com";urlRepository;constructor(t){this.axios=d.create({baseURL:this.domain}),this.axios.interceptors.response.use(t=>t.data,t=>Promise.reject(t)),this.urlRepository=t.getRepository(f)}async getDownloadUrl({access_token:t,drive_id:e,file_id:a,ua:r}){const i=this.getKey({access_token:t,file_id:a,ua:r});try{for(;h[i];)await new Promise(t=>setTimeout(t,100));h[i]=!0;const s=await this.urlRepository.findOne({where:{key:i,fail:l(2),expire:u(Date.now())},order:{fail:"ASC"}});if(s)return s.fail>1?(await this.urlRepository.delete({key:i}),""):s.dlink;const{url:n,expiration:o}=await this.fetchDownloadInfo({access_token:t,drive_id:e,file_id:a,ua:r});if(n)return await this.urlRepository.save({dlink:n,key:i,speed:100,expire:new Date(o).getTime()}),n;throw new Error("ali fetch download url failed")}finally{delete h[i]}}async getArrayBuffer(t){return await this.multipleGetArrayBuffer({...t,times:0})}async multipleGetArrayBuffer({access_token:t,drive_id:e,file_id:a,start:r,end:i,abort_controller:s,times:n=0,ua:o}){const c=await this.getDownloadUrl({access_token:t,drive_id:e,file_id:a,ua:o});if(!c)throw new Error("获取下载链接失败");try{const t=i-r+1,e=Math.max(1e3*Math.ceil(t/1024/50),3e3);return(await d.get(c,{headers:{Range:`bytes=${r}-${i}`,"User-Agent":o,Accept:"application/octet-stream"},responseType:"arraybuffer",timeout:e,signal:s?.signal})).data}catch(s){if("AbortError"===s.name)return;const d=this.getKey({access_token:t,file_id:a,ua:o});if(await this.urlRepository.increment({key:d,dlink:c},"fail",1),await new Promise(t=>setTimeout(t,1e3)),(n||0)<3)return await this.multipleGetArrayBuffer({access_token:t,drive_id:e,file_id:a,start:r,end:i,times:(n||0)+1,ua:o});throw s}}async fetchDownloadInfo({access_token:t,drive_id:e,file_id:a,ua:r}){const i={Authorization:`Bearer ${t}`,"User-Agent":r},{url:s,expiration:n}=await this.axios.post("/adrive/v1.0/openFile/getDownloadUrl",{drive_id:e,file_id:a},{headers:i});return{url:s,expiration:n}}getKey({access_token:t,file_id:e,ua:a}){const r=a?encodeURIComponent(a):"ua";return`ali-${encodeURIComponent(t)}-${r}-${e}`}},p=t=>(w||(w=new _(t)),w);import{getDownloadBaiduUA as y}from"@soga/baidu-ua";var m={},k=({part_md5:t,start:e,end:a})=>`${t}-${e}-${a}`;async function g(t){const e=function(t){const{start:e,end:a,total:r}=t,i=Math.max(a-e+1,0);if(2==t.host_vip_type){const r=Math.ceil(i/1572864),s=Math.ceil(i/r),n=[];for(let i=0;i<r;i++){const r=e+i*s,o=Math.min(a,r+s-1);n.push({...t,start:r,end:o,auto_ua:!1})}return{ranges:n,threads:2}}{const s=Math.ceil(i/131072),n=Math.ceil(i/s),o=[];for(let r=0;r<s;r++){const i=e+r*n,s=Math.min(a,i+n-1);o.push({...t,start:i,end:s,auto_ua:!0})}return{ranges:o,threads:Math.min(r<52428800?8:6,s)}}}(t),a=await async function(t,e){const{ranges:a,threads:r}=t,i=[...a],s=async()=>{if(!i.length)return;const t=i.shift();if(!t)return;const a=k(t);if(!m[a]?.data){const r=c(e),{access_token:i,fs_id:s,host_id:n}=t,{ua:o,finish:d}=await y(n,t.total),l=await r.getArrayBuffer({access_token:i,fs_id:s,start:t.start,end:t.end,ua:t.auto_ua?o:t.ua});if(d(),!l||0===l.byteLength)return;m[a]={updated:Date.now(),data:Buffer.from(l)}}await s()},n=[];for(let t=0;t<r;t++)n.push(s());await Promise.all(n);let o=!0;const d=a.map(t=>{const e=k(t);return m[e]?.data?m[e].data:(o=!1,null)});if(o){const t=Buffer.concat(d);return a.forEach(t=>{const e=k(t);delete m[e]}),t}return null}(e,t.dataSource);return a}async function b(t){const e=function(t){const{start:e,end:a}=t,r=Math.max(a-e+1,0),i=Math.ceil(r/1572864),s=Math.ceil(r/i),n=[];for(let r=0;r<i;r++){const i=e+r*s,o=Math.min(a,i+s-1);n.push({...t,start:i,end:o})}return{ranges:n,threads:2}}(t),a=await async function(t,e){const{ranges:a,threads:r}=t,i=[...a],s=async()=>{if(!i.length)return;const t=i.shift();if(!t)return;const a=k(t);if(!m[a]?.data){const r=p(e),{access_token:i,drive_id:s,file_id:n}=t,o=await r.getArrayBuffer({access_token:i,drive_id:s,file_id:n,start:t.start,end:t.end,ua:t.ua,abort_controller:t.abort_controller});m[a]={updated:Date.now(),data:Buffer.from(o)}}await s()},n=[];for(let t=0;t<r;t++)n.push(s());await Promise.all(n);let o=!0;const c=a.map(t=>{const e=k(t);return m[e]?.data?m[e].data:(o=!1,null)});if(o){const t=Buffer.concat(c);return a.forEach(t=>{const e=k(t);delete m[e]}),t}return null}(e,t.dataSource);return a}import{getDb as R}from"@soga/lowdb";import{readFile as x,remove as D,writeFile as v}from"fs-extra";import{resolve as B}from"path";var A=class{folder_path;cache_path;db;cache_type;constructor(t,e){this.folder_path=t,this.cache_type=e,this.cache_path=B(t,`${e}_cache`)}async init(){const t=B(this.cache_path,"cache.json");this.db=await R(t,{})}async read(t){return this.db.data[t]}async write(t,e){this.db.data[t]=e,await this.db.write()}async delete(t){delete this.db.data[t],await this.db.write()}async writeBuffer(t,e){const a=B(this.cache_path,t);await v(a,e),this.db.data[t]=!0,await this.db.write()}async readBuffer(t){if(!this.db.data[t])return null;const e=B(this.cache_path,t);return await x(e)}async deleteBuffer(t){if(!this.db.data[t])return;delete this.db.data[t],await this.db.write();const e=B(this.cache_path,t);try{await D(e)}catch(t){}}},U={},$=0,M=async(t,e)=>{if(U[t]){for(;!U[t].inited;)await new Promise(t=>setTimeout(t,20));U[t].updated_at=Date.now();const e=Date.now();return e-$>18e5&&($=e,Object.keys(U).forEach(t=>{e-U[t].updated_at>288e5&&delete U[t]})),U[t].store}const a=new A(t,e);return U[t]={updated_at:Date.now(),store:a,inited:!1},await a.init(),U[t].inited=!0,a},E=async({id:t,dataSource:e,record_id:a,start:r,end:i,total:s,part_md5:n,access_token:o,host_id:c,host_vip_type:d,fs_id:l,cache_folder:u,abort_controller:f},h)=>{let w=null;const _=t||k({part_md5:n,start:r,end:i});if(u){w=await M(u,h);const t=await w.readBuffer(_);if(t)return t}const p=await g({dataSource:e,record_id:a,start:r,end:i,total:s,access_token:o,host_id:c,host_vip_type:d,fs_id:l,part_md5:n,abort_controller:f});return p&&w&&await w.writeBuffer(_,p),p},S=async t=>await E(t,"preview"),T=async t=>await E(t,"download"),P=async({id:t,dataSource:e,record_id:a,start:r,end:i,total:s,part_md5:n,access_token:o,host_id:c,host_vip_type:d,drive_id:l,file_id:u,cache_folder:f,abort_controller:h},w)=>{const _=t||k({part_md5:n,start:r,end:i});let p=null;if(f){p=await M(f,w);const t=await p.readBuffer(_);if(t)return t}const y=await b({dataSource:e,record_id:a,start:r,end:i,total:s,access_token:o,host_id:c,host_vip_type:d,drive_id:l,file_id:u,part_md5:n,abort_controller:h});return y&&p&&await p.writeBuffer(_,y),y},C=async t=>await P(t,"preview"),G=async t=>await P(t,"download");export{_ as AliFetcher,o as BaiduFetcher,b as fetchAliBuffer,g as fetchBaiduBuffer,G as getAliDownloadBuffer,p as getAliFetcher,C as getAliPreviewBuffer,T as getBaiduDownloadBuffer,c as getBaiduFetcher,S as getBaiduPreviewBuffer};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/fetcher",
-  "version": "0.3.0",
+  "version": "1.3.1",
   "publishConfig": {
     "access": "public"
   },
@@ -14,20 +14,22 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@soga/baidu-ua": "^0.3.0",
-    "@soga/bridge": "^0.3.0",
-    "@soga/entities": "^0.3.0",
     "axios": "^1.9.0",
-    "typeorm": "^0.3.24"
+    "fs-extra": "^11.3.2",
+    "typeorm": "^0.3.24",
+    "@soga/baidu-ua": "^1.0.0",
+    "@soga/entities": "^1.0.0",
+    "@soga/lowdb": "^1.0.2"
+  },
+  "devDependencies": {
+    "@types/node": "^24.5.2",
+    "rimraf": "^6.0.1",
+    "terser": "^5.43.1",
+    "tsup": "^8.5.0",
+    "typescript": "^5.8.3",
+    "@soga/typescript-config": "^1.0.0"
   },
   "scripts": {
-    "build": "rimraf dist && tsc && ts-node ./scripts/minify",
-    "minify": "ts-node ./scripts/minify",
-    "demo_backup": "ts-node ./demo/demo.ts",
-    "demo": "ts-node ./demo/demo.ts",
-    "worker": "tsc && ts-node ./demo/worker.ts",
-    "test": "jest",
-    "dev": "ts-node ./src/main.ts",
-    "lint": "eslint . --ext .ts"
+    "build": "rimraf dist && tsup src/main.ts --format cjs,esm --dts --minify terser"
   }
 }

package/README.md

@@ -1 +0,0 @@
-# downloader

package/dist/ali/fetch-buffer.d.ts

@@ -1,2 +0,0 @@
-import { AliBufferParams } from '../common/types';
-export declare function fetchAliBuffer(params: AliBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/ali/fetch-buffer.js

@@ -1,83 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchAliBuffer = fetchAliBuffer;
-const fetcher_1 = require("./fetcher");
-const common_1 = require("../common/common");
-async function fetchAliBuffer(params) {
-    const rangeData = calculateRangeData(params);
-    const buffer = await parallelFetchBuffer(rangeData, params.dataSource);
-    return buffer;
-}
-async function parallelFetchBuffer(params, dataSource) {
-    const { ranges, threads } = params;
-    const queues = [...ranges];
-    const loop = async () => {
-        if (!queues.length)
-            return;
-        const item = queues.shift();
-        if (!item)
-            return;
-        const key = (0, common_1.getKey)(item);
-        if (!common_1.bufferCache[key]?.data) {
-            const fetcher = (0, fetcher_1.getAliFetcher)(dataSource);
-            const { access_token, drive_id, file_id } = item;
-            const ab = await fetcher.getArrayBuffer({
-                access_token,
-                drive_id,
-                file_id,
-                start: item.start,
-                end: item.end,
-                ua: item.ua,
-                abort_controller: item.abort_controller,
-            });
-            common_1.bufferCache[key] = {
-                updated: Date.now(),
-                data: Buffer.from(ab),
-            };
-        }
-        await loop();
-    };
-    const arr = [];
-    for (let i = 0; i < threads; i++) {
-        arr.push(loop());
-    }
-    await Promise.all(arr);
-    let completed = true;
-    const buffers = ranges.map((item) => {
-        const key = (0, common_1.getKey)(item);
-        if (common_1.bufferCache[key]?.data) {
-            return common_1.bufferCache[key].data;
-        }
-        completed = false;
-        return null;
-    });
-    if (completed) {
-        const buffer = Buffer.concat(buffers);
-        ranges.forEach((item) => {
-            const key = (0, common_1.getKey)(item);
-            delete common_1.bufferCache[key];
-        });
-        return buffer;
-    }
-    return null;
-}
-function calculateRangeData(params) {
-    const { start, end } = params;
-    const size = Math.max(end - start + 1, 0);
-    const segments = Math.ceil(size / 1572864);
-    const bytes = Math.ceil(size / segments);
-    const ranges = [];
-    for (let i = 0; i < segments; i++) {
-        const the_start = start + i * bytes;
-        const the_end = Math.min(end, the_start + bytes - 1);
-        ranges.push({
-            ...params,
-            start: the_start,
-            end: the_end,
-        });
-    }
-    return {
-        ranges,
-        threads: 2,
-    };
-}

package/dist/ali/fetcher.d.ts

@@ -1,28 +0,0 @@
-import { AxiosInstance } from 'axios';
-import { DataSource, Repository } from 'typeorm';
-import { FetchUrl } from '@soga/entities';
-export declare class AliFetcher {
-    axios: AxiosInstance;
-    private domain;
-    protected urlRepository: Repository<FetchUrl>;
-    constructor(dataSource: DataSource);
-    getDownloadUrl({ access_token, drive_id, file_id, ua, }: {
-        access_token: string;
-        drive_id: string;
-        file_id: string;
-        ua?: string;
-    }): Promise<string>;
-    getArrayBuffer(params: {
-        access_token: string;
-        drive_id: string;
-        file_id: string;
-        start: number;
-        end: number;
-        ua?: string;
-        abort_controller?: AbortController;
-    }): Promise<ArrayBuffer>;
-    private multipleGetArrayBuffer;
-    private fetchDownloadInfo;
-    private getKey;
-}
-export declare const getAliFetcher: (dataSource: DataSource) => AliFetcher;

package/dist/ali/fetcher.js

@@ -1,145 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAliFetcher = exports.AliFetcher = void 0;
-const axios_1 = __importDefault(require("axios"));
-const typeorm_1 = require("typeorm");
-const entities_1 = require("@soga/entities");
-const processMap = {};
-let instance = null;
-class AliFetcher {
-    axios;
-    domain = 'https://openapi.alipan.com';
-    urlRepository;
-    constructor(dataSource) {
-        this.axios = axios_1.default.create({
-            baseURL: this.domain,
-        });
-        this.axios.interceptors.response.use((response) => {
-            return response.data;
-        }, (error) => {
-            return Promise.reject(error);
-        });
-        this.urlRepository = dataSource.getRepository(entities_1.FetchUrl);
-    }
-    async getDownloadUrl({ access_token, drive_id, file_id, ua, }) {
-        const key = this.getKey({ access_token, file_id, ua });
-        try {
-            while (processMap[key]) {
-                await new Promise((resolve) => setTimeout(resolve, 100));
-            }
-            processMap[key] = true;
-            const row = await this.urlRepository.findOne({
-                where: { key, fail: (0, typeorm_1.LessThan)(2), expire: (0, typeorm_1.MoreThan)(Date.now()) },
-                order: {
-                    fail: 'ASC',
-                },
-            });
-            if (row) {
-                if (row.fail > 1) {
-                    await this.urlRepository.delete({ key });
-                    return '';
-                }
-                return row.dlink;
-            }
-            const { url, expiration } = await this.fetchDownloadInfo({
-                access_token,
-                drive_id,
-                file_id,
-                ua,
-            });
-            if (url) {
-                await this.urlRepository.save({
-                    dlink: url,
-                    key,
-                    speed: 100,
-                    expire: new Date(expiration).getTime(),
-                });
-                return url;
-            }
-            throw new Error('ali fetch download url failed');
-        }
-        finally {
-            delete processMap[key];
-        }
-    }
-    async getArrayBuffer(params) {
-        return await this.multipleGetArrayBuffer({
-            ...params,
-            times: 0,
-        });
-    }
-    async multipleGetArrayBuffer({ access_token, drive_id, file_id, start, end, abort_controller, times = 0, ua, }) {
-        const dlink = await this.getDownloadUrl({
-            access_token,
-            drive_id,
-            file_id,
-            ua,
-        });
-        if (!dlink) {
-            throw new Error('获取下载链接失败');
-        }
-        try {
-            const length = end - start + 1;
-            const duration = Math.max(Math.ceil(length / 1024 / 50) * 1000, 3000);
-            const res = await axios_1.default.get(dlink, {
-                headers: {
-                    Range: `bytes=${start}-${end}`,
-                    'User-Agent': ua,
-                    Accept: 'application/octet-stream',
-                },
-                responseType: 'arraybuffer',
-                timeout: duration,
-                signal: abort_controller?.signal,
-            });
-            return res.data;
-        }
-        catch (err) {
-            if (err.name === 'AbortError')
-                return;
-            const key = this.getKey({ access_token, file_id, ua });
-            await this.urlRepository.increment({ key, dlink }, 'fail', 1);
-            await new Promise((resolve) => setTimeout(resolve, 1000));
-            if ((times || 0) < 3) {
-                return await this.multipleGetArrayBuffer({
-                    access_token,
-                    drive_id,
-                    file_id,
-                    start,
-                    end,
-                    times: (times || 0) + 1,
-                    ua,
-                });
-            }
-            throw err;
-        }
-    }
-    async fetchDownloadInfo({ access_token, drive_id, file_id, ua, }) {
-        const headers = {
-            Authorization: `Bearer ${access_token}`,
-            'User-Agent': ua,
-        };
-        const { url, expiration } = await this.axios.post('/adrive/v1.0/openFile/getDownloadUrl', {
-            drive_id,
-            file_id,
-        }, {
-            headers,
-        });
-        return { url, expiration };
-    }
-    getKey({ access_token, file_id, ua, }) {
-        const userAgent = ua ? encodeURIComponent(ua) : 'ua';
-        const accessToken = encodeURIComponent(access_token);
-        return `ali-${accessToken}-${userAgent}-${file_id}`;
-    }
-}
-exports.AliFetcher = AliFetcher;
-const getAliFetcher = (dataSource) => {
-    if (!instance) {
-        instance = new AliFetcher(dataSource);
-    }
-    return instance;
-};
-exports.getAliFetcher = getAliFetcher;

package/dist/ali/get-buffer.d.ts

@@ -1,3 +0,0 @@
-import { AliPreviewBufferParams } from '../common/types';
-export declare const getAliPreviewBuffer: (params: AliPreviewBufferParams) => Promise<Buffer<ArrayBuffer>>;
-export declare const getAliDownloadBuffer: (params: AliPreviewBufferParams) => Promise<Buffer<ArrayBuffer>>;

package/dist/ali/get-buffer.js

@@ -1,42 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAliDownloadBuffer = exports.getAliPreviewBuffer = void 0;
-const common_1 = require("../common/common");
-const fetch_buffer_1 = require("./fetch-buffer");
-const low_store_1 = require("../common/low-store");
-const getAliBuffer = async ({ dataSource, record_id, start, end, total, part_md5, access_token, host_id, host_vip_type, drive_id, file_id, cache_folder, abort_controller, }, cache_type) => {
-    const key = (0, common_1.getKey)({ part_md5, start, end });
-    let store = null;
-    if (cache_folder) {
-        store = await (0, low_store_1.getLowStore)(cache_folder, cache_type);
-        const bufferCache = await store.readBuffer(key);
-        if (bufferCache) {
-            return bufferCache;
-        }
-    }
-    const buffer = await (0, fetch_buffer_1.fetchAliBuffer)({
-        dataSource,
-        record_id,
-        start,
-        end,
-        total,
-        access_token,
-        host_id,
-        host_vip_type,
-        drive_id,
-        file_id,
-        part_md5,
-        abort_controller,
-    });
-    if (buffer && store) {
-        await store.writeBuffer(key, buffer);
-    }
-    return buffer;
-};
-const getAliPreviewBuffer = async (params) => {
-    console.log('getAliPreviewBuffer params:', params);
-    return await getAliBuffer(params, 'preview');
-};
-exports.getAliPreviewBuffer = getAliPreviewBuffer;
-const getAliDownloadBuffer = async (params) => await getAliBuffer(params, 'download');
-exports.getAliDownloadBuffer = getAliDownloadBuffer;

package/dist/baidu/fetch-buffer.d.ts

@@ -1,2 +0,0 @@
-import { BaiduBufferParams } from '../common/types';
-export declare function fetchBaiduBuffer(params: BaiduBufferParams): Promise<Buffer<ArrayBuffer>>;

package/dist/baidu/fetch-buffer.js

@@ -1,109 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchBaiduBuffer = fetchBaiduBuffer;
-const baidu_ua_1 = require("@soga/baidu-ua");
-const fetcher_1 = require("./fetcher");
-const common_1 = require("../common/common");
-async function fetchBaiduBuffer(params) {
-    const rangeData = calculateRangeData(params);
-    const buffer = await parallelFetchBuffer(rangeData, params.dataSource);
-    return buffer;
-}
-async function parallelFetchBuffer(params, dataSource) {
-    const { ranges, threads } = params;
-    const queues = [...ranges];
-    const loop = async () => {
-        if (!queues.length)
-            return;
-        const item = queues.shift();
-        if (!item)
-            return;
-        const key = (0, common_1.getKey)(item);
-        if (!common_1.bufferCache[key]?.data) {
-            const fetcher = (0, fetcher_1.getBaiduFetcher)(dataSource);
-            const { access_token, fs_id, host_id } = item;
-            const { ua, finishBaiduUA } = await (0, baidu_ua_1.getBaiduUA)(host_id, item.total);
-            const ab = await fetcher.getArrayBuffer({
-                access_token,
-                fs_id,
-                start: item.start,
-                end: item.end,
-                ua: item.auto_ua ? ua : item.ua,
-            });
-            finishBaiduUA();
-            if (!ab || ab.byteLength === 0) {
-                return;
-            }
-            common_1.bufferCache[key] = {
-                updated: Date.now(),
-                data: Buffer.from(ab),
-            };
-        }
-        await loop();
-    };
-    const arr = [];
-    for (let i = 0; i < threads; i++) {
-        arr.push(loop());
-    }
-    await Promise.all(arr);
-    let completed = true;
-    const buffers = ranges.map((item) => {
-        const key = (0, common_1.getKey)(item);
-        if (common_1.bufferCache[key]?.data) {
-            return common_1.bufferCache[key].data;
-        }
-        completed = false;
-        return null;
-    });
-    if (completed) {
-        const buffer = Buffer.concat(buffers);
-        ranges.forEach((item) => {
-            const key = (0, common_1.getKey)(item);
-            delete common_1.bufferCache[key];
-        });
-        return buffer;
-    }
-    return null;
-}
-function calculateRangeData(params) {
-    const { start, end, total } = params;
-    const size = Math.max(end - start + 1, 0);
-    if (params.host_vip_type == 2) {
-        const segments = Math.ceil(size / 1572864);
-        const bytes = Math.ceil(size / segments);
-        const ranges = [];
-        for (let i = 0; i < segments; i++) {
-            const the_start = start + i * bytes;
-            const the_end = Math.min(end, the_start + bytes - 1);
-            ranges.push({
-                ...params,
-                start: the_start,
-                end: the_end,
-                auto_ua: false,
-            });
-        }
-        return {
-            ranges,
-            threads: 2,
-        };
-    }
-    else {
-        const segments = Math.ceil(size / 131072);
-        const bytes = Math.ceil(size / segments);
-        const ranges = [];
-        for (let i = 0; i < segments; i++) {
-            const the_start = start + i * bytes;
-            const the_end = Math.min(end, the_start + bytes - 1);
-            ranges.push({
-                ...params,
-                start: the_start,
-                end: the_end,
-                auto_ua: true,
-            });
-        }
-        return {
-            ranges,
-            threads: Math.min(total < 50 * 1024 * 1024 ? 8 : 6, segments),
-        };
-    }
-}

package/dist/baidu/fetcher.d.ts

@@ -1,35 +0,0 @@
-import { AxiosInstance } from 'axios';
-import { DataSource, Repository } from 'typeorm';
-import { FetchUrl } from '@soga/entities';
-export declare class BaiduFetcher {
-    axios: AxiosInstance;
-    private domain;
-    protected urlRepository: Repository<FetchUrl>;
-    constructor(dataSource: DataSource);
-    getDownloadUrl({ access_token, fs_id, ua, }: {
-        access_token: string;
-        fs_id: number;
-        ua?: string;
-    }): Promise<string>;
-    getArrayBuffer(params: {
-        access_token: string;
-        fs_id: number;
-        start: number;
-        end: number;
-        abort_controller?: AbortController;
-        ua?: string;
-    }): Promise<ArrayBuffer>;
-    getDownloadUrlByRange({ access_token, fs_id, start, end, ua, }: {
-        access_token: string;
-        fs_id: number;
-        start: number;
-        end: number;
-        ua?: string;
-    }): Promise<string>;
-    private cleanExpired;
-    private fetchBestDlink;
-    private fetchDownloadUrl;
-    private multipleGetArrayBuffer;
-    private getKey;
-}
-export declare const getBaiduFetcher: (dataSource: DataSource) => BaiduFetcher;

package/dist/baidu/fetcher.js

@@ -1,243 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBaiduFetcher = exports.BaiduFetcher = void 0;
-const axios_1 = __importDefault(require("axios"));
-const typeorm_1 = require("typeorm");
-const entities_1 = require("@soga/entities");
-const processMap = {};
-let instance = null;
-let lastCleanTime = 0;
-class BaiduFetcher {
-    axios;
-    domain = 'https://pan.baidu.com';
-    urlRepository;
-    constructor(dataSource) {
-        this.axios = axios_1.default.create({
-            baseURL: this.domain,
-        });
-        this.axios.interceptors.response.use((response) => {
-            return response.data;
-        }, (error) => {
-            return Promise.reject(error);
-        });
-        this.urlRepository = dataSource.getRepository(entities_1.FetchUrl);
-    }
-    async getDownloadUrl({ access_token, fs_id, ua, }) {
-        const url = await this.getDownloadUrlByRange({
-            access_token,
-            fs_id,
-            start: 0,
-            end: 0,
-            ua,
-        });
-        return url;
-    }
-    async getArrayBuffer(params) {
-        await this.cleanExpired();
-        return await this.multipleGetArrayBuffer({
-            ...params,
-            times: 0,
-        });
-    }
-    async getDownloadUrlByRange({ access_token, fs_id, start = 0, end = 0, ua, }) {
-        const key = this.getKey({ fs_id, access_token, ua });
-        try {
-            while (processMap[key]) {
-                await new Promise((resolve) => setTimeout(resolve, 100));
-            }
-            processMap[key] = true;
-            const row = await this.urlRepository.findOne({
-                where: {
-                    key,
-                    speed: (0, typeorm_1.MoreThan)(30),
-                    expire: (0, typeorm_1.MoreThan)(Date.now()),
-                },
-                order: {
-                    speed: 'DESC',
-                    fail: 'ASC',
-                },
-            });
-            if (row) {
-                if (row.fail > 10) {
-                    await this.urlRepository.delete({ key });
-                    return '';
-                }
-                return row.dlink;
-            }
-            const { dlink, speed } = await this.fetchBestDlink({
-                access_token,
-                fs_id,
-                start,
-                end,
-                ua,
-            });
-            if (dlink) {
-                await this.urlRepository.save({
-                    dlink,
-                    key,
-                    speed,
-                    expire: Date.now() + 7 * 3600000,
-                });
-                return dlink;
-            }
-            throw new Error('baidu fetch download url failed');
-        }
-        finally {
-            delete processMap[key];
-        }
-    }
-    async cleanExpired() {
-        if (Date.now() - lastCleanTime > 1800000) {
-            await this.urlRepository.delete({
-                expire: (0, typeorm_1.LessThan)(Date.now()),
-            });
-            lastCleanTime = Date.now();
-        }
-    }
-    async fetchBestDlink({ fs_id, start = 0, end = 0, ua, access_token, }) {
-        const errorDomains = [];
-        let download_link = '';
-        const url = await this.fetchDownloadUrl({ fs_id, access_token, ua });
-        if (!url) {
-            return {
-                dlink: '',
-                speed: 0,
-            };
-        }
-        const to = Math.min(start + 127, end);
-        const headers = {
-            'User-Agent': ua,
-            Range: `bytes=${start}-${to}`,
-        };
-        const bytes = to - start + 1;
-        const times = {
-            start: 0,
-            end: 0,
-        };
-        for (let i = 0; i < 50; i++) {
-            const res = await axios_1.default.get(url, {
-                headers,
-                maxRedirects: 0,
-                validateStatus: (status) => status >= 200 && status < 303,
-            });
-            const dlink = res.headers.location;
-            if (!dlink) {
-                return {
-                    dlink: '',
-                    speed: 0,
-                };
-            }
-            const $url = new URL(dlink);
-            const { hostname } = $url;
-            if (errorDomains.includes(hostname)) {
-                continue;
-            }
-            download_link = dlink;
-            try {
-                times.start = Date.now();
-                await axios_1.default.get(dlink, {
-                    headers,
-                    timeout: 2000,
-                });
-                times.end = Date.now();
-                return {
-                    dlink,
-                    speed: (1000 * bytes) / (times.end - times.start),
-                };
-            }
-            catch (err) {
-                errorDomains.push(hostname);
-                await new Promise((resolve) => setTimeout(resolve, 200));
-                continue;
-            }
-        }
-        return {
-            dlink: download_link,
-            speed: (1000 * bytes) / (times.end - times.start),
-        };
-    }
-    async fetchDownloadUrl({ fs_id, ua = 'pan.baidu.com', access_token, }) {
-        const res = await this.axios.get('/rest/2.0/xpan/multimedia', {
-            params: {
-                access_token,
-                dlink: '1',
-                method: 'filemetas',
-                fsids: JSON.stringify([fs_id]),
-            },
-            headers: {
-                'User-Agent': ua,
-            },
-        });
-        const { errno, list } = res || {};
-        if (errno != 0) {
-            throw new Error(`获取下载链接失败: ${errno}`);
-        }
-        if (!list || !list.length) {
-            throw new Error('获取下载链接失败');
-        }
-        const url = `${list[0].dlink}&access_token=${access_token}`;
-        return url;
-    }
-    async multipleGetArrayBuffer({ access_token, fs_id, start, end, abort_controller, times = 0, ua, }) {
-        const dlink = await this.getDownloadUrlByRange({
-            access_token,
-            fs_id,
-            start,
-            end,
-            ua,
-        });
-        if (!dlink) {
-            throw new Error('获取下载链接失败');
-        }
-        try {
-            const length = end - start + 1;
-            const duration = Math.max(Math.ceil(length / 1024 / 10) * 1000, 3000);
-            const res = await axios_1.default.get(dlink, {
-                headers: {
-                    Range: `bytes=${start}-${end}`,
-                    'User-Agent': ua,
-                    Accept: 'application/octet-stream',
-                },
-                signal: abort_controller?.signal,
-                responseType: 'arraybuffer',
-                timeout: duration,
-            });
-            return res.data;
-        }
-        catch (err) {
-            if (err.name === 'AbortError')
-                return;
-            const key = this.getKey({ fs_id, access_token, ua });
-            await this.urlRepository.increment({ key, dlink }, 'fail', 1);
-            await new Promise((resolve) => setTimeout(resolve, 1000));
-            if ((times || 0) < 3) {
-                return await this.multipleGetArrayBuffer({
-                    access_token,
-                    fs_id,
-                    start,
-                    end,
-                    times: (times || 0) + 1,
-                    abort_controller,
-                    ua,
-                });
-            }
-            throw err;
-        }
-    }
-    getKey({ access_token, fs_id, ua, }) {
-        const userAgent = ua ? encodeURIComponent(ua) : 'ua';
-        const accessToken = encodeURIComponent(access_token);
-        return `baidu-${accessToken}-${userAgent}-${fs_id}`;
-    }
-}
-exports.BaiduFetcher = BaiduFetcher;
-const getBaiduFetcher = (dataSource) => {
-    if (!instance) {
-        instance = new BaiduFetcher(dataSource);
-    }
-    return instance;
-};
-exports.getBaiduFetcher = getBaiduFetcher;

package/dist/baidu/get-buffer.d.ts

@@ -1,3 +0,0 @@
-import { BaiduPreviewBufferParams } from '../common/types';
-export declare const getBaiduPreviewBuffer: (params: BaiduPreviewBufferParams) => Promise<Buffer<ArrayBuffer>>;
-export declare const getBaiduDownloadBuffer: (params: BaiduPreviewBufferParams) => Promise<Buffer<ArrayBuffer>>;

package/dist/baidu/get-buffer.js

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBaiduDownloadBuffer = exports.getBaiduPreviewBuffer = void 0;
-const common_1 = require("../common/common");
-const fetch_buffer_1 = require("./fetch-buffer");
-const low_store_1 = require("../common/low-store");
-const entities_1 = require("@soga/entities");
-const getBaiduBuffer = async ({ dataSource, record_id, start, end, total, part_md5, access_token, host_id, host_vip_type, fs_id, cache_folder, abort_controller, }, cache_type) => {
-    const rep = dataSource.getRepository(entities_1.FetchUrl);
-    const data = await rep.find({});
-    let store = null;
-    const key = (0, common_1.getKey)({ part_md5, start, end });
-    if (cache_folder) {
-        store = await (0, low_store_1.getLowStore)(cache_folder, cache_type);
-        const bufferCache = await store.readBuffer(key);
-        if (bufferCache) {
-            return bufferCache;
-        }
-    }
-    const buffer = await (0, fetch_buffer_1.fetchBaiduBuffer)({
-        dataSource,
-        record_id,
-        start,
-        end,
-        total,
-        access_token,
-        host_id,
-        host_vip_type,
-        fs_id,
-        part_md5,
-        abort_controller,
-    });
-    if (buffer && store) {
-        await store.writeBuffer(key, buffer);
-    }
-    return buffer;
-};
-const getBaiduPreviewBuffer = async (params) => await getBaiduBuffer(params, 'preview');
-exports.getBaiduPreviewBuffer = getBaiduPreviewBuffer;
-const getBaiduDownloadBuffer = async (params) => await getBaiduBuffer(params, 'download');
-exports.getBaiduDownloadBuffer = getBaiduDownloadBuffer;

package/dist/common/common.d.ts

@@ -1,9 +0,0 @@
-export declare const bufferCache: Record<string, {
-    updated: number;
-    data: Buffer<ArrayBuffer>;
-}>;
-export declare const getKey: ({ part_md5, start, end, }: {
-    part_md5: string;
-    start: number;
-    end: number;
-}) => string;

package/dist/common/common.js

@@ -1,8 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getKey = exports.bufferCache = void 0;
-exports.bufferCache = {};
-const getKey = ({ part_md5, start, end, }) => {
-    return `${part_md5}-${start}-${end}`;
-};
-exports.getKey = getKey;

package/dist/common/low-store.d.ts

@@ -1,19 +0,0 @@
-import { LowType } from '@soga/bridge';
-import { CacheType } from './types';
-export declare class LowStore {
-    folder_path: string;
-    cache_path: string;
-    db: LowType<DbDataType>;
-    cache_type: CacheType;
-    constructor(folder_path: string, cache_type: CacheType);
-    init(): Promise<void>;
-    read(key: string): Promise<boolean>;
-    write(key: string, value: boolean): Promise<void>;
-    delete(key: string): Promise<void>;
-    writeBuffer(key: string, buffer: Buffer<ArrayBuffer | ArrayBufferLike>): Promise<void>;
-    readBuffer(key: string): Promise<NonSharedBuffer>;
-    deleteBuffer(key: string): Promise<void>;
-}
-export declare const getLowStore: (folder_path: string, cache_type: CacheType) => Promise<LowStore>;
-type DbDataType = Record<string, boolean>;
-export {};

package/dist/common/low-store.js

@@ -1,91 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLowStore = exports.LowStore = void 0;
-const bridge_1 = require("@soga/bridge");
-const fs_extra_1 = require("fs-extra");
-const path_1 = require("path");
-class LowStore {
-    folder_path;
-    cache_path;
-    db;
-    cache_type;
-    constructor(folder_path, cache_type) {
-        this.folder_path = folder_path;
-        this.cache_type = cache_type;
-        this.cache_path = (0, path_1.resolve)(folder_path, `${cache_type}_cache`);
-    }
-    async init() {
-        const dbPath = (0, path_1.resolve)(this.cache_path, 'cache.json');
-        this.db = await (0, bridge_1.getDb)(dbPath, {});
-    }
-    async read(key) {
-        return this.db.data[key];
-    }
-    async write(key, value) {
-        this.db.data[key] = value;
-        await this.db.write();
-    }
-    async delete(key) {
-        delete this.db.data[key];
-        await this.db.write();
-    }
-    async writeBuffer(key, buffer) {
-        const chunkPath = (0, path_1.resolve)(this.cache_path, key);
-        await (0, fs_extra_1.writeFile)(chunkPath, buffer);
-        this.db.data[key] = true;
-        await this.db.write();
-    }
-    async readBuffer(key) {
-        const exist = this.db.data[key];
-        if (!exist) {
-            return null;
-        }
-        const chunkPath = (0, path_1.resolve)(this.cache_path, key);
-        const data = await (0, fs_extra_1.readFile)(chunkPath);
-        return data;
-    }
-    async deleteBuffer(key) {
-        const exist = this.db.data[key];
-        if (!exist)
-            return;
-        delete this.db.data[key];
-        await this.db.write();
-        const chunkPath = (0, path_1.resolve)(this.cache_path, key);
-        try {
-            await (0, fs_extra_1.remove)(chunkPath);
-        }
-        catch (e) {
-        }
-    }
-}
-exports.LowStore = LowStore;
-const storeMap = {};
-let lastCleanTime = 0;
-const getLowStore = async (folder_path, cache_type) => {
-    if (storeMap[folder_path]) {
-        while (!storeMap[folder_path].inited) {
-            await new Promise((resolve) => setTimeout(resolve, 20));
-        }
-        storeMap[folder_path].updated_at = Date.now();
-        const now = Date.now();
-        if (now - lastCleanTime > 30 * 60000) {
-            lastCleanTime = now;
-            Object.keys(storeMap).forEach((key) => {
-                if (now - storeMap[key].updated_at > 8 * 60 * 60 * 1000) {
-                    delete storeMap[key];
-                }
-            });
-        }
-        return storeMap[folder_path].store;
-    }
-    const store = new LowStore(folder_path, cache_type);
-    storeMap[folder_path] = {
-        updated_at: Date.now(),
-        store,
-        inited: false,
-    };
-    await store.init();
-    storeMap[folder_path].inited = true;
-    return store;
-};
-exports.getLowStore = getLowStore;

package/dist/common/types.d.ts

@@ -1,29 +0,0 @@
-import { DataSource } from 'typeorm';
-interface CommonBufferParams {
-    dataSource: DataSource;
-    record_id: number;
-    part_md5: string;
-    start: number;
-    end: number;
-    total: number;
-    host_id: number;
-    host_vip_type: number;
-    access_token: string;
-    abort_controller?: AbortController;
-    ua?: string;
-}
-export interface BaiduBufferParams extends CommonBufferParams {
-    fs_id: number;
-}
-export interface AliBufferParams extends CommonBufferParams {
-    drive_id: string;
-    file_id: string;
-}
-export interface BaiduPreviewBufferParams extends BaiduBufferParams {
-    cache_folder: string;
-}
-export interface AliPreviewBufferParams extends AliBufferParams {
-    cache_folder: string;
-}
-export type CacheType = 'preview' | 'download';
-export {};

package/dist/common/types.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });