instasave-sdk 1.2.7 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +148 -0
- package/README.md +219 -2
- package/dist/browser/crawler-manager.d.ts +6 -0
- package/dist/browser/crawler-manager.d.ts.map +1 -1
- package/dist/browser/crawler-manager.js +29 -1
- package/dist/browser/crawler-manager.js.map +1 -1
- package/dist/core/media-scraper.d.ts +6 -0
- package/dist/core/media-scraper.d.ts.map +1 -1
- package/dist/core/media-scraper.js +16 -2
- package/dist/core/media-scraper.js.map +1 -1
- package/dist/download/download-engine.d.ts +32 -0
- package/dist/download/download-engine.d.ts.map +1 -0
- package/dist/download/download-engine.js +177 -0
- package/dist/download/download-engine.js.map +1 -0
- package/dist/download/download-manager.d.ts +18 -0
- package/dist/download/download-manager.d.ts.map +1 -0
- package/dist/download/download-manager.js +232 -0
- package/dist/download/download-manager.js.map +1 -0
- package/dist/download/download-scrape.d.ts +32 -0
- package/dist/download/download-scrape.d.ts.map +1 -0
- package/dist/download/download-scrape.js +159 -0
- package/dist/download/download-scrape.js.map +1 -0
- package/dist/download/index.d.ts +4 -0
- package/dist/download/index.d.ts.map +1 -0
- package/dist/download/index.js +25 -0
- package/dist/download/index.js.map +1 -0
- package/dist/download/types.d.ts +48 -0
- package/dist/download/types.d.ts.map +1 -0
- package/dist/download/types.js +6 -0
- package/dist/download/types.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/platforms/instagram/index.d.ts.map +1 -1
- package/dist/platforms/instagram/index.js +48 -0
- package/dist/platforms/instagram/index.js.map +1 -1
- package/dist/utils/browser/browser-path-resolver.d.ts +29 -0
- package/dist/utils/browser/browser-path-resolver.d.ts.map +1 -0
- package/dist/utils/browser/browser-path-resolver.js +76 -0
- package/dist/utils/browser/browser-path-resolver.js.map +1 -0
- package/dist/utils/config/config-parser.d.ts +2 -0
- package/dist/utils/config/config-parser.d.ts.map +1 -1
- package/dist/utils/config/config-parser.js.map +1 -1
- package/instasave.config.example +18 -20
- package/package.json +6 -3
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Download engine - HTTP download with retries
|
|
3
|
+
*/
|
|
4
|
+
interface DownloadResultInternal {
|
|
5
|
+
success: boolean;
|
|
6
|
+
bytes: number;
|
|
7
|
+
path: string;
|
|
8
|
+
error?: string;
|
|
9
|
+
}
|
|
10
|
+
interface DownloadFileParams {
|
|
11
|
+
url: string;
|
|
12
|
+
destPath: string;
|
|
13
|
+
filename: string;
|
|
14
|
+
onProgress?: (percent: number) => void;
|
|
15
|
+
timeout?: number;
|
|
16
|
+
}
|
|
17
|
+
export declare class DownloadEngine {
|
|
18
|
+
private readonly tempDir;
|
|
19
|
+
private readonly defaultTimeout;
|
|
20
|
+
constructor();
|
|
21
|
+
download(params: DownloadFileParams): Promise<DownloadResultInternal>;
|
|
22
|
+
private downloadWithRetry;
|
|
23
|
+
private performDownload;
|
|
24
|
+
getFileNameFromUrl(url: string): string;
|
|
25
|
+
getFileFormat(url: string): string;
|
|
26
|
+
getFileHash(filePath: string): Promise<string>;
|
|
27
|
+
private ensureTempDir;
|
|
28
|
+
private cleanupTemp;
|
|
29
|
+
private sleep;
|
|
30
|
+
}
|
|
31
|
+
export {};
|
|
32
|
+
//# sourceMappingURL=download-engine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download-engine.d.ts","sourceRoot":"","sources":["../../src/download/download-engine.ts"],"names":[],"mappings":"AAAA;;GAEG;AAQH,UAAU,sBAAsB;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,kBAAkB;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;;IAMlC,QAAQ,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,sBAAsB,CAAC;YAM7D,iBAAiB;YAqCjB,eAAe;IA6D7B,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAWvC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAM5B,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWpD,OAAO,CAAC,aAAa;IAMrB,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,KAAK;CAGd"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Download engine - HTTP download with retries
|
|
4
|
+
*/
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
39
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
40
|
+
};
|
|
41
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
+
exports.DownloadEngine = void 0;
|
|
43
|
+
const axios_1 = __importDefault(require("axios"));
|
|
44
|
+
const fs = __importStar(require("fs"));
|
|
45
|
+
const path = __importStar(require("path"));
|
|
46
|
+
const crypto = __importStar(require("crypto"));
|
|
47
|
+
const logger_1 = require("../utils/logger");
|
|
48
|
+
class DownloadEngine {
|
|
49
|
+
constructor() {
|
|
50
|
+
this.tempDir = './temp';
|
|
51
|
+
this.defaultTimeout = 30000;
|
|
52
|
+
this.ensureTempDir();
|
|
53
|
+
}
|
|
54
|
+
async download(params) {
|
|
55
|
+
const { url, destPath, filename, onProgress, timeout = this.defaultTimeout } = params;
|
|
56
|
+
return this.downloadWithRetry(url, destPath, filename, 3, timeout, onProgress);
|
|
57
|
+
}
|
|
58
|
+
async downloadWithRetry(url, destPath, filename, maxRetries, timeout, onProgress) {
|
|
59
|
+
let attempt = 0;
|
|
60
|
+
let lastError = null;
|
|
61
|
+
while (attempt < maxRetries) {
|
|
62
|
+
attempt++;
|
|
63
|
+
const retryDelay = Math.pow(2, attempt - 1) * 1000;
|
|
64
|
+
try {
|
|
65
|
+
logger_1.log.debug(`⬇️ Downloading ${filename} (attempt ${attempt}/${maxRetries})`);
|
|
66
|
+
return await this.performDownload(url, destPath, timeout, onProgress);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
lastError = error;
|
|
70
|
+
logger_1.log.warn(`⬇️ Download attempt ${attempt} failed: ${lastError.message}`);
|
|
71
|
+
if (attempt < maxRetries) {
|
|
72
|
+
logger_1.log.debug(`⬇️ Retrying in ${retryDelay}ms...`);
|
|
73
|
+
await this.sleep(retryDelay);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
success: false,
|
|
79
|
+
bytes: 0,
|
|
80
|
+
path: '',
|
|
81
|
+
error: lastError?.message || 'Unknown error'
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
async performDownload(url, destPath, timeout, onProgress) {
|
|
85
|
+
const tempFilePath = path.join(this.tempDir, `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`);
|
|
86
|
+
const response = await (0, axios_1.default)({
|
|
87
|
+
method: 'GET',
|
|
88
|
+
url: url,
|
|
89
|
+
responseType: 'stream',
|
|
90
|
+
timeout: timeout,
|
|
91
|
+
maxRedirects: 5,
|
|
92
|
+
headers: {
|
|
93
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
94
|
+
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
|
|
95
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
96
|
+
'Referer': 'https://www.instagram.com/'
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
const totalBytes = parseInt(response.headers['content-length'] || '0', 10);
|
|
100
|
+
let downloadedBytes = 0;
|
|
101
|
+
return new Promise((resolve, reject) => {
|
|
102
|
+
const writer = fs.createWriteStream(tempFilePath);
|
|
103
|
+
response.data.on('data', (chunk) => {
|
|
104
|
+
downloadedBytes += chunk.length;
|
|
105
|
+
if (totalBytes > 0 && onProgress) {
|
|
106
|
+
const percent = Math.round((downloadedBytes / totalBytes) * 100);
|
|
107
|
+
onProgress(percent);
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
response.data.pipe(writer);
|
|
111
|
+
writer.on('finish', () => {
|
|
112
|
+
try {
|
|
113
|
+
fs.renameSync(tempFilePath, destPath);
|
|
114
|
+
this.cleanupTemp('');
|
|
115
|
+
resolve({
|
|
116
|
+
success: true,
|
|
117
|
+
bytes: downloadedBytes,
|
|
118
|
+
path: destPath
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
this.cleanupTemp(tempFilePath);
|
|
123
|
+
reject(error);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
writer.on('error', (error) => {
|
|
127
|
+
this.cleanupTemp(tempFilePath);
|
|
128
|
+
reject(error);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
getFileNameFromUrl(url) {
|
|
133
|
+
try {
|
|
134
|
+
const urlObj = new URL(url);
|
|
135
|
+
const pathname = urlObj.pathname;
|
|
136
|
+
const filename = path.basename(pathname);
|
|
137
|
+
return filename || `image-${Date.now()}.jpg`;
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return `image-${Date.now()}.jpg`;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
getFileFormat(url) {
|
|
144
|
+
const filename = this.getFileNameFromUrl(url);
|
|
145
|
+
const ext = path.extname(filename).toLowerCase();
|
|
146
|
+
return ext.replace('.', '') || 'jpg';
|
|
147
|
+
}
|
|
148
|
+
async getFileHash(filePath) {
|
|
149
|
+
return new Promise((resolve, reject) => {
|
|
150
|
+
const hash = crypto.createHash('md5');
|
|
151
|
+
const stream = fs.createReadStream(filePath);
|
|
152
|
+
stream.on('data', (data) => hash.update(data));
|
|
153
|
+
stream.on('end', () => resolve(hash.digest('hex')));
|
|
154
|
+
stream.on('error', reject);
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
ensureTempDir() {
|
|
158
|
+
if (!fs.existsSync(this.tempDir)) {
|
|
159
|
+
fs.mkdirSync(this.tempDir, { recursive: true });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
cleanupTemp(filePath) {
|
|
163
|
+
try {
|
|
164
|
+
if (filePath && fs.existsSync(filePath)) {
|
|
165
|
+
fs.unlinkSync(filePath);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
logger_1.log.debug(`Failed to cleanup temp file: ${error}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
sleep(ms) {
|
|
173
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
exports.DownloadEngine = DownloadEngine;
|
|
177
|
+
//# sourceMappingURL=download-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download-engine.js","sourceRoot":"","sources":["../../src/download/download-engine.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,kDAA6C;AAC7C,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,4CAAsC;AAiBtC,MAAa,cAAc;IAIzB;QAHiB,YAAO,GAAG,QAAQ,CAAC;QACnB,mBAAc,GAAG,KAAK,CAAC;QAGtC,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAA0B;QACvC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,MAAM,CAAC;QAEtF,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACjF,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,GAAW,EACX,QAAgB,EAChB,QAAgB,EAChB,UAAkB,EAClB,OAAe,EACf,UAAsC;QAEtC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAiB,IAAI,CAAC;QAEnC,OAAO,OAAO,GAAG,UAAU,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;YACV,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;YAEnD,IAAI,CAAC;gBACH,YAAG,CAAC,KAAK,CAAC,mBAAmB,QAAQ,aAAa,OAAO,IAAI,UAAU,GAAG,CAAC,CAAC;gBAC5E,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAc,CAAC;gBAC3B,YAAG,CAAC,IAAI,CAAC,wBAAwB,OAAO,YAAY,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBAEzE,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;oBACzB,YAAG,CAAC,KAAK,CAAC,mBAAmB,UAAU,OAAO,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,CAAC;YACR,IAAI,EAAE,EAAE;YACR,KAAK,EAAE,SAAS,EAAE,OAAO,IAAI,eAAe;SAC7C,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,GAAW,EACX,QAAgB,EAChB,OAAe,EACf,UAAsC;QAEtC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAEzG,MAAM,QAAQ,GAAkB,MAAM,IAAA,eAAK,EAAC;YAC1C,MAAM,EAAE,KAAK;YACb,GAAG,EAAE,GAAG;YACR,YAAY,EAAE,QAAQ;YACtB,OAAO,EAAE,OAAO;YAChB,YAAY,EAAE,CAAC;YACf,OAAO,EAAE;gBACP,YAAY,EAAE,iHAAiH;gBAC/H,QAAQ,EAAE,yCAAyC;gBACnD,iBAAiB,EAAE,gBAAgB;gBACnC,SAAS,EAAE,4BAA4B;aACxC;SACF,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3E,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBACzC,eAAe,IAAI,KAAK,CAAC,MAAM,CAAC;gBAChC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;oBACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;oBACjE,UAAU,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3B,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;gBACvB,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACtC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;oBAErB,OAAO,CAAC;wBACN,OAAO,EAAE,IAAI;wBACb,KAAK,EAAE,eAAe;wBACtB,IAAI,EAAE,QAAQ;qBACf,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;oBAC/B,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;gBAC/B,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB,CAAC,GAAW;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,OAAO,QAAQ,IAAI,SAAS,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;QACnC,CAAC;IACH,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAgB;QAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAE7C,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,QAAgB;QAClC,IAAI,CAAC;YACH,IAAI,QAAQ,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAG,CAAC,KAAK,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF;AA/JD,wCA+JC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Download manager - main API for downloading images
|
|
3
|
+
*/
|
|
4
|
+
import { MediaData } from '../types';
|
|
5
|
+
import { DownloadOptions, DownloadResult } from './types';
|
|
6
|
+
export declare function downloadImages(data: MediaData, options: DownloadOptions): Promise<DownloadResult>;
|
|
7
|
+
export declare class DownloadManager {
|
|
8
|
+
private engine;
|
|
9
|
+
private duplicateCache;
|
|
10
|
+
constructor();
|
|
11
|
+
downloadImages(data: MediaData, options: DownloadOptions): Promise<DownloadResult>;
|
|
12
|
+
generateFilename(data: MediaData, index: number, pattern: string, ext: string): string;
|
|
13
|
+
private checkDuplicate;
|
|
14
|
+
private hashString;
|
|
15
|
+
private sanitizeFilename;
|
|
16
|
+
private createDirectory;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=download-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download-manager.d.ts","sourceRoot":"","sources":["../../src/download/download-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EACL,eAAe,EACf,cAAc,EAGf,MAAM,SAAS,CAAC;AAEjB,wBAAsB,cAAc,CAClC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,cAAc,CAAC,CAGzB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,cAAc,CAAc;;IAO9B,cAAc,CAClB,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,cAAc,CAAC;IAmI1B,gBAAgB,CACd,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,GACV,MAAM;YAsBK,cAAc;IAmC5B,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;CAKxB"}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Download manager - main API for downloading images
|
|
4
|
+
*/
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.DownloadManager = void 0;
|
|
40
|
+
exports.downloadImages = downloadImages;
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const path = __importStar(require("path"));
|
|
43
|
+
const logger_1 = require("../utils/logger");
|
|
44
|
+
const download_engine_1 = require("./download-engine");
|
|
45
|
+
async function downloadImages(data, options) {
|
|
46
|
+
const manager = new DownloadManager();
|
|
47
|
+
return manager.downloadImages(data, options);
|
|
48
|
+
}
|
|
49
|
+
class DownloadManager {
|
|
50
|
+
constructor() {
|
|
51
|
+
this.engine = new download_engine_1.DownloadEngine();
|
|
52
|
+
this.duplicateCache = new Set();
|
|
53
|
+
}
|
|
54
|
+
async downloadImages(data, options) {
|
|
55
|
+
const startTime = Date.now();
|
|
56
|
+
const { savePath, getFile = 'save', filename = '{username}_{postid}_{index}.{ext}', skipDuplicates = false, createSubfolders = false, onProgress, timeout = 30000 } = options;
|
|
57
|
+
if (!data.media || data.media.length === 0) {
|
|
58
|
+
return {
|
|
59
|
+
success: true,
|
|
60
|
+
total: 0,
|
|
61
|
+
downloaded: 0,
|
|
62
|
+
failed: 0,
|
|
63
|
+
skipped: 0,
|
|
64
|
+
files: [],
|
|
65
|
+
errors: [],
|
|
66
|
+
totalBytes: 0,
|
|
67
|
+
duration: Date.now() - startTime
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
let actualPath = savePath;
|
|
71
|
+
if (createSubfolders && data.profile_name) {
|
|
72
|
+
actualPath = path.join(savePath, this.sanitizeFilename(data.profile_name));
|
|
73
|
+
}
|
|
74
|
+
this.createDirectory(actualPath);
|
|
75
|
+
const files = [];
|
|
76
|
+
const errors = [];
|
|
77
|
+
let downloaded = 0;
|
|
78
|
+
let failed = 0;
|
|
79
|
+
let skipped = 0;
|
|
80
|
+
let totalBytes = 0;
|
|
81
|
+
for (let i = 0; i < data.media.length; i++) {
|
|
82
|
+
const mediaItem = data.media[i];
|
|
83
|
+
const current = i + 1;
|
|
84
|
+
const total = data.media.length;
|
|
85
|
+
const percent = Math.round((current / total) * 100);
|
|
86
|
+
if (onProgress) {
|
|
87
|
+
onProgress(percent, current, total);
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
const ext = this.engine.getFileFormat(mediaItem.url);
|
|
91
|
+
const generatedFilename = this.generateFilename(data, i, filename, ext);
|
|
92
|
+
const destPath = path.join(actualPath, generatedFilename);
|
|
93
|
+
if (getFile === false) {
|
|
94
|
+
files.push({
|
|
95
|
+
index: i,
|
|
96
|
+
url: mediaItem.url,
|
|
97
|
+
path: destPath,
|
|
98
|
+
filename: generatedFilename,
|
|
99
|
+
bytes: 0,
|
|
100
|
+
format: ext
|
|
101
|
+
});
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (skipDuplicates) {
|
|
105
|
+
const tempPath = path.join('./temp', `check-${Date.now()}`);
|
|
106
|
+
const isDuplicate = await this.checkDuplicate(mediaItem.url, tempPath);
|
|
107
|
+
if (isDuplicate) {
|
|
108
|
+
skipped++;
|
|
109
|
+
logger_1.log.debug(`⬇️ Skipped duplicate: ${generatedFilename}`);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const result = await this.engine.download({
|
|
114
|
+
url: mediaItem.url,
|
|
115
|
+
destPath: destPath,
|
|
116
|
+
filename: generatedFilename,
|
|
117
|
+
timeout: timeout
|
|
118
|
+
});
|
|
119
|
+
if (result.success) {
|
|
120
|
+
downloaded++;
|
|
121
|
+
totalBytes += result.bytes;
|
|
122
|
+
files.push({
|
|
123
|
+
index: i,
|
|
124
|
+
url: mediaItem.url,
|
|
125
|
+
path: result.path,
|
|
126
|
+
filename: generatedFilename,
|
|
127
|
+
bytes: result.bytes,
|
|
128
|
+
format: ext
|
|
129
|
+
});
|
|
130
|
+
logger_1.log.debug(`⬇️ Downloaded: ${generatedFilename}`);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
failed++;
|
|
134
|
+
errors.push({
|
|
135
|
+
index: i,
|
|
136
|
+
url: mediaItem.url,
|
|
137
|
+
code: 'DOWNLOAD_FAILED',
|
|
138
|
+
message: result.error || 'Unknown download error'
|
|
139
|
+
});
|
|
140
|
+
logger_1.log.warn(`⬇️ Failed to download: ${generatedFilename}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
failed++;
|
|
145
|
+
errors.push({
|
|
146
|
+
index: i,
|
|
147
|
+
url: mediaItem.url,
|
|
148
|
+
code: 'ERROR',
|
|
149
|
+
message: error instanceof Error ? error.message : String(error)
|
|
150
|
+
});
|
|
151
|
+
logger_1.log.error(`⬇️ Error downloading ${mediaItem.url}:`, error);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
success: failed === 0,
|
|
156
|
+
total: data.media.length,
|
|
157
|
+
downloaded,
|
|
158
|
+
failed,
|
|
159
|
+
skipped,
|
|
160
|
+
files,
|
|
161
|
+
errors,
|
|
162
|
+
totalBytes,
|
|
163
|
+
duration: Date.now() - startTime
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
generateFilename(data, index, pattern, ext) {
|
|
167
|
+
const postId = data.post_id || 'unknown';
|
|
168
|
+
const username = this.sanitizeFilename(data.profile_name || 'user');
|
|
169
|
+
const platform = data.platform || 'instagram';
|
|
170
|
+
const timestamp = Date.now();
|
|
171
|
+
const paddedIndex = (index + 1).toString().padStart(2, '0');
|
|
172
|
+
let filename = pattern
|
|
173
|
+
.replace('{username}', username)
|
|
174
|
+
.replace('{postid}', postId)
|
|
175
|
+
.replace('{index}', paddedIndex)
|
|
176
|
+
.replace('{platform}', platform)
|
|
177
|
+
.replace('{timestamp}', timestamp.toString())
|
|
178
|
+
.replace('{ext}', ext);
|
|
179
|
+
if (!path.extname(filename)) {
|
|
180
|
+
filename = `${filename}.${ext}`;
|
|
181
|
+
}
|
|
182
|
+
return filename;
|
|
183
|
+
}
|
|
184
|
+
async checkDuplicate(url, tempPath) {
|
|
185
|
+
const urlHash = this.hashString(url);
|
|
186
|
+
if (this.duplicateCache.has(urlHash)) {
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const downloadResult = await this.engine.download({
|
|
191
|
+
url: url,
|
|
192
|
+
destPath: tempPath,
|
|
193
|
+
filename: 'temp-check',
|
|
194
|
+
timeout: 5000
|
|
195
|
+
});
|
|
196
|
+
if (!downloadResult.success) {
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
const hash = await this.engine.getFileHash(tempPath);
|
|
200
|
+
this.duplicateCache.add(hash);
|
|
201
|
+
this.duplicateCache.add(urlHash);
|
|
202
|
+
if (fs.existsSync(tempPath)) {
|
|
203
|
+
fs.unlinkSync(tempPath);
|
|
204
|
+
}
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
hashString(str) {
|
|
212
|
+
let hash = 0;
|
|
213
|
+
for (let i = 0; i < str.length; i++) {
|
|
214
|
+
const char = str.charCodeAt(i);
|
|
215
|
+
hash = ((hash << 5) - hash) + char;
|
|
216
|
+
hash = hash & hash;
|
|
217
|
+
}
|
|
218
|
+
return hash.toString(16);
|
|
219
|
+
}
|
|
220
|
+
sanitizeFilename(filename) {
|
|
221
|
+
return filename
|
|
222
|
+
.replace(/[^a-zA-Z0-9\-_.]/g, '_')
|
|
223
|
+
.substring(0, 100);
|
|
224
|
+
}
|
|
225
|
+
createDirectory(dirPath) {
|
|
226
|
+
if (!fs.existsSync(dirPath)) {
|
|
227
|
+
fs.mkdirSync(dirPath, { recursive: true });
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
exports.DownloadManager = DownloadManager;
|
|
232
|
+
//# sourceMappingURL=download-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download-manager.js","sourceRoot":"","sources":["../../src/download/download-manager.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAcH,wCAMC;AAlBD,uCAAyB;AACzB,2CAA6B;AAC7B,4CAAsC;AAEtC,uDAAmD;AAQ5C,KAAK,UAAU,cAAc,CAClC,IAAe,EACf,OAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,OAAO,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED,MAAa,eAAe;IAI1B;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,gCAAc,EAAE,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,IAAe,EACf,OAAwB;QAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EACJ,QAAQ,EACR,OAAO,GAAG,MAAM,EAChB,QAAQ,GAAG,mCAAmC,EAC9C,cAAc,GAAG,KAAK,EACtB,gBAAgB,GAAG,KAAK,EACxB,UAAU,EACV,OAAO,GAAG,KAAK,EAChB,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,EAAE;gBACV,UAAU,EAAE,CAAC;gBACb,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACjC,CAAC;QACJ,CAAC;QAED,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,IAAI,gBAAgB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;QAEjC,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAEpD,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrD,MAAM,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;gBACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;gBAE1D,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;oBACtB,KAAK,CAAC,IAAI,CAAC;wBACT,KAAK,EAAE,CAAC;wBACR,GAAG,EAAE,SAAS,CAAC,GAAG;wBAClB,IAAI,EAAE,QAAQ;wBACd,QAAQ,EAAE,iBAAiB;wBAC3B,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,GAAG;qBACZ,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBAED,IAAI,cAAc,EAAE,CAAC;oBACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC5D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBACvE,IAAI,WAAW,EAAE,CAAC;wBAChB,OAAO,EAAE,CAAC;wBACV,YAAG,CAAC,KAAK,CAAC,0BAA0B,iBAAiB,EAAE,CAAC,CAAC;wBACzD,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;oBACxC,GAAG,EAAE,SAAS,CAAC,GAAG;oBAClB,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,iBAAiB;oBAC3B,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,UAAU,EAAE,CAAC;oBACb,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC;oBAC3B,KAAK,CAAC,IAAI,CAAC;wBACT,KAAK,EAAE,CAAC;wBACR,GAAG,EAAE,SAAS,CAAC,GAAG;wBAClB,IAAI,EAAE,MAAM,CAAC,IAAI;wBACjB,QAAQ,EAAE,iBAAiB;wBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;wBACnB,MAAM,EAAE,GAAG;qBACZ,CAAC,CAAC;oBACH,YAAG,CAAC,KAAK,CAAC,mBAAmB,iBAAiB,EAAE,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,CAAC;oBACT,MAAM,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,GAAG,EAAE,SAAS,CAAC,GAAG;wBAClB,IAAI,EAAE,iBAAiB;wBACvB,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,wBAAwB;qBAClD,CAAC,CAAC;oBACH,YAAG,CAAC,IAAI,CAAC,2BAA2B,iBAAiB,EAAE,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,CAAC;oBACR,GAAG,EAAE,SAAS,CAAC,GAAG;oBAClB,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAChE,CAAC,CAAC;gBACH,YAAG,CAAC,KAAK,CAAC,yBAAyB,SAAS,CAAC,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,MAAM,KAAK,CAAC;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACxB,UAAU;YACV,MAAM;YACN,OAAO;YACP,KAAK;YACL,MAAM;YACN,UAAU;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACjC,CAAC;IACJ,CAAC;IAED,gBAAgB,CACd,IAAe,EACf,KAAa,EACb,OAAe,EACf,GAAW;QAEX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,IAAI,SAAS,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAE5D,IAAI,QAAQ,GAAG,OAAO;aACnB,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC;aAC/B,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC;aAC3B,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC;aAC/B,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC;aAC/B,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC;aAC5C,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,QAAQ,GAAG,GAAG,QAAQ,IAAI,GAAG,EAAE,CAAC;QAClC,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,GAAW,EACX,QAAgB;QAEhB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChD,GAAG,EAAE,GAAG;gBACR,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;gBAC5B,OAAO,KAAK,CAAC;YACf,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEjC,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,GAAW;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAEO,gBAAgB,CAAC,QAAgB;QACvC,OAAO,QAAQ;aACZ,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC;aACjC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvB,CAAC;IAEO,eAAe,CAAC,OAAe;QACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;CACF;AAlOD,0CAkOC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Download Scrape API
|
|
3
|
+
* Download image from URL with optional Base64 conversion
|
|
4
|
+
*
|
|
5
|
+
* Flow:
|
|
6
|
+
* - useBase64 = true: Download → Base64 → File
|
|
7
|
+
* - useBase64 = false: Direct Download → File
|
|
8
|
+
*/
|
|
9
|
+
export interface DownloadScrapeOptions {
|
|
10
|
+
useBase64?: boolean;
|
|
11
|
+
outputPath?: string;
|
|
12
|
+
filename?: string;
|
|
13
|
+
timeout?: number;
|
|
14
|
+
retries?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface DownloadScrapeResult {
|
|
17
|
+
success: boolean;
|
|
18
|
+
url: string;
|
|
19
|
+
localPath: string;
|
|
20
|
+
filename: string;
|
|
21
|
+
format: string;
|
|
22
|
+
bytes: number;
|
|
23
|
+
error?: string;
|
|
24
|
+
}
|
|
25
|
+
export declare function downloadScrape(url: string, options?: DownloadScrapeOptions): Promise<DownloadScrapeResult>;
|
|
26
|
+
export declare class DownloadScrapeEngine {
|
|
27
|
+
private defaultOptions;
|
|
28
|
+
constructor(defaultOptions?: DownloadScrapeOptions);
|
|
29
|
+
download(url: string, options?: DownloadScrapeOptions): Promise<DownloadScrapeResult>;
|
|
30
|
+
downloadBatch(urls: string[], options?: DownloadScrapeOptions): Promise<DownloadScrapeResult[]>;
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=download-scrape.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download-scrape.d.ts","sourceRoot":"","sources":["../../src/download/download-scrape.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AA6BD,wBAAsB,cAAc,CAClC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC,CAqF/B;AAGD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,cAAc,CAAwB;gBAElC,cAAc,GAAE,qBAA0B;IAIhD,QAAQ,CACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,CAAC;IAI1B,aAAa,CACjB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,oBAAoB,EAAE,CAAC;CAKnC"}
|