@sovovs/bycli 2.1.16 → 2.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
7
|
*/
|
|
8
|
+
import * as crypto from 'node:crypto';
|
|
8
9
|
import * as fs from 'node:fs';
|
|
9
10
|
import * as path from 'node:path';
|
|
10
11
|
import TurndownService from 'turndown';
|
|
@@ -238,11 +239,38 @@ function defaultDetectImageExt(url) {
|
|
|
238
239
|
const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
|
|
239
240
|
return extMatch ? extMatch[1] : 'jpg';
|
|
240
241
|
}
|
|
242
|
+
/** Random 8-char lowercase hex segment used in downloaded image filenames. */
|
|
243
|
+
function randomImageId() {
|
|
244
|
+
return crypto.randomBytes(4).toString('hex');
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Build a collision-free `img_<8 hex>.<ext>` filename inside `imgDir`.
|
|
248
|
+
*
|
|
249
|
+
* `taken` guards against collisions between images downloaded concurrently in
|
|
250
|
+
* the same run, since those files do not exist on disk yet when the name is
|
|
251
|
+
* picked. The on-disk check covers files from earlier runs into the same dir.
|
|
252
|
+
*/
|
|
253
|
+
function buildImageFilename(imgDir, ext, taken) {
|
|
254
|
+
for (let attempt = 0; attempt < 10; attempt++) {
|
|
255
|
+
const filename = `img_${randomImageId()}.${ext}`;
|
|
256
|
+
if (taken.has(filename))
|
|
257
|
+
continue;
|
|
258
|
+
if (fs.existsSync(path.join(imgDir, filename)))
|
|
259
|
+
continue;
|
|
260
|
+
taken.add(filename);
|
|
261
|
+
return filename;
|
|
262
|
+
}
|
|
263
|
+
// Astronomically unlikely; fall back to a longer id rather than overwriting.
|
|
264
|
+
const filename = `img_${randomImageId()}${randomImageId()}.${ext}`;
|
|
265
|
+
taken.add(filename);
|
|
266
|
+
return filename;
|
|
267
|
+
}
|
|
241
268
|
async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
242
269
|
const urlMap = {};
|
|
243
270
|
if (imgUrls.length === 0)
|
|
244
271
|
return urlMap;
|
|
245
272
|
const detect = detectExt || defaultDetectImageExt;
|
|
273
|
+
const takenFilenames = new Set();
|
|
246
274
|
// Deduplicate image URLs
|
|
247
275
|
const seen = new Set();
|
|
248
276
|
const uniqueUrls = imgUrls.filter(url => {
|
|
@@ -253,13 +281,12 @@ async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
|
253
281
|
});
|
|
254
282
|
for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
|
|
255
283
|
const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
|
|
256
|
-
const results = await Promise.all(batch.map(async (rawUrl
|
|
257
|
-
const index = i + j + 1;
|
|
284
|
+
const results = await Promise.all(batch.map(async (rawUrl) => {
|
|
258
285
|
let imgUrl = rawUrl;
|
|
259
286
|
if (imgUrl.startsWith('//'))
|
|
260
287
|
imgUrl = `https:${imgUrl}`;
|
|
261
288
|
const ext = detect(imgUrl);
|
|
262
|
-
const filename =
|
|
289
|
+
const filename = buildImageFilename(imgDir, ext, takenFilenames);
|
|
263
290
|
const filepath = path.join(imgDir, filename);
|
|
264
291
|
try {
|
|
265
292
|
const result = await httpDownload(imgUrl, filepath, {
|
package/package.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|