@sovovs/bycli 2.1.15 → 2.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
7
|
*/
|
|
8
|
+
import * as crypto from 'node:crypto';
|
|
8
9
|
import * as fs from 'node:fs';
|
|
9
10
|
import * as path from 'node:path';
|
|
10
11
|
import TurndownService from 'turndown';
|
|
@@ -238,11 +239,38 @@ function defaultDetectImageExt(url) {
|
|
|
238
239
|
const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
|
|
239
240
|
return extMatch ? extMatch[1] : 'jpg';
|
|
240
241
|
}
|
|
242
|
+
/** Random 8-char lowercase hex segment used in downloaded image filenames. */
|
|
243
|
+
function randomImageId() {
|
|
244
|
+
return crypto.randomBytes(4).toString('hex');
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Build a collision-free `img_<8 hex>.<ext>` filename inside `imgDir`.
|
|
248
|
+
*
|
|
249
|
+
* `taken` guards against collisions between images downloaded concurrently in
|
|
250
|
+
* the same run, since those files do not exist on disk yet when the name is
|
|
251
|
+
* picked. The on-disk check covers files from earlier runs into the same dir.
|
|
252
|
+
*/
|
|
253
|
+
function buildImageFilename(imgDir, ext, taken) {
|
|
254
|
+
for (let attempt = 0; attempt < 10; attempt++) {
|
|
255
|
+
const filename = `img_${randomImageId()}.${ext}`;
|
|
256
|
+
if (taken.has(filename))
|
|
257
|
+
continue;
|
|
258
|
+
if (fs.existsSync(path.join(imgDir, filename)))
|
|
259
|
+
continue;
|
|
260
|
+
taken.add(filename);
|
|
261
|
+
return filename;
|
|
262
|
+
}
|
|
263
|
+
// Astronomically unlikely; fall back to a longer id rather than overwriting.
|
|
264
|
+
const filename = `img_${randomImageId()}${randomImageId()}.${ext}`;
|
|
265
|
+
taken.add(filename);
|
|
266
|
+
return filename;
|
|
267
|
+
}
|
|
241
268
|
async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
242
269
|
const urlMap = {};
|
|
243
270
|
if (imgUrls.length === 0)
|
|
244
271
|
return urlMap;
|
|
245
272
|
const detect = detectExt || defaultDetectImageExt;
|
|
273
|
+
const takenFilenames = new Set();
|
|
246
274
|
// Deduplicate image URLs
|
|
247
275
|
const seen = new Set();
|
|
248
276
|
const uniqueUrls = imgUrls.filter(url => {
|
|
@@ -253,13 +281,12 @@ async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
|
253
281
|
});
|
|
254
282
|
for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
|
|
255
283
|
const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
|
|
256
|
-
const results = await Promise.all(batch.map(async (rawUrl
|
|
257
|
-
const index = i + j + 1;
|
|
284
|
+
const results = await Promise.all(batch.map(async (rawUrl) => {
|
|
258
285
|
let imgUrl = rawUrl;
|
|
259
286
|
if (imgUrl.startsWith('//'))
|
|
260
287
|
imgUrl = `https:${imgUrl}`;
|
|
261
288
|
const ext = detect(imgUrl);
|
|
262
|
-
const filename =
|
|
289
|
+
const filename = buildImageFilename(imgDir, ext, takenFilenames);
|
|
263
290
|
const filepath = path.join(imgDir, filename);
|
|
264
291
|
try {
|
|
265
292
|
const result = await httpDownload(imgUrl, filepath, {
|
|
@@ -336,13 +363,20 @@ export async function downloadArticle(data, options) {
|
|
|
336
363
|
// Shape: `# Title\n[> meta\n...]\n---\n\n<markdown>` — exactly one blank
|
|
337
364
|
// line separates every section, so we never produce ≥3 consecutive newlines.
|
|
338
365
|
const headerValue = (value) => secureMarkdown ? escapeMarkdownText(value) : value;
|
|
366
|
+
// The source URL must not go through escapeMarkdownText: escaping `. _ - ( ) #`
|
|
367
|
+
// and entity-encoding `&` leaves a link that can no longer be copied or
|
|
368
|
+
// followed. safeHttpUrl already allowlists http(s) and percent-encodes `<`/`>`
|
|
369
|
+
// via URL normalization, so the autolink cannot be closed early; an empty
|
|
370
|
+
// return means a hostile or malformed URL and the line is dropped entirely.
|
|
371
|
+
const headerUrl = (value) => secureMarkdown ? safeHttpUrl(value) : value;
|
|
339
372
|
const headerLines = [`# ${headerValue(data.title)}`];
|
|
340
373
|
if (data.author)
|
|
341
374
|
headerLines.push(`> ${labels.author}: ${headerValue(data.author)}`);
|
|
342
375
|
if (data.publishTime)
|
|
343
376
|
headerLines.push(`> ${labels.publishTime}: ${headerValue(data.publishTime)}`);
|
|
344
|
-
|
|
345
|
-
|
|
377
|
+
const sourceUrl = data.sourceUrl ? headerUrl(data.sourceUrl) : '';
|
|
378
|
+
if (sourceUrl)
|
|
379
|
+
headerLines.push(`> ${labels.sourceUrl}: <${sourceUrl}>`);
|
|
346
380
|
const frontmatter = headerLines.join('\n') + '\n\n---\n\n';
|
|
347
381
|
const fullContent = frontmatter + markdown;
|
|
348
382
|
const size = Buffer.byteLength(fullContent, 'utf-8');
|
package/package.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|