@link-assistant/web-capture 1.4.2 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/Dockerfile +26 -21
- package/README.md +92 -72
- package/bin/web-capture.js +259 -51
- package/package.json +5 -1
- package/src/archive.js +19 -6
- package/src/batch.js +3 -3
- package/src/extract-images.js +120 -0
- package/src/gdocs.js +8 -6
- package/src/lib.js +57 -2
- package/src/markdown.js +7 -2
- package/src/postprocess.js +2 -4
- package/tests/unit/batch.test.js +4 -4
- package/tests/unit/extract-images.test.js +233 -0
- package/tests/unit/gdocs.test.js +75 -0
- package/tests/unit/heading-image.test.js +49 -0
- package/tests/unit/postprocess.test.js +4 -2
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
const BASE64_IMAGE_REGEX =
|
|
6
|
+
/!\[([^\]]*)\]\(data:image\/(png|jpeg|jpg|gif|webp|svg\+xml);base64,([^)]+)\)/gi;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* @param {string} markdown - Markdown content with data URI images
|
|
10
|
+
* @param {string} outputDir - Directory where the markdown file is being written
|
|
11
|
+
* @param {Object} [options]
|
|
12
|
+
* @param {string} [options.imagesDir='images'] - Subdirectory name for images
|
|
13
|
+
* @returns {{markdown: string, extracted: number}}
|
|
14
|
+
*/
|
|
15
|
+
export function extractAndSaveImages(markdown, outputDir, options = {}) {
|
|
16
|
+
const { imagesDir = 'images' } = options;
|
|
17
|
+
const imagesPath = path.resolve(outputDir, imagesDir);
|
|
18
|
+
|
|
19
|
+
const images = [];
|
|
20
|
+
|
|
21
|
+
const updatedMarkdown = markdown.replace(
|
|
22
|
+
BASE64_IMAGE_REGEX,
|
|
23
|
+
(match, altText, mimeExt, base64Data) => {
|
|
24
|
+
const ext =
|
|
25
|
+
mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
|
|
26
|
+
|
|
27
|
+
let buffer;
|
|
28
|
+
try {
|
|
29
|
+
buffer = Buffer.from(base64Data, 'base64');
|
|
30
|
+
} catch {
|
|
31
|
+
return match;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const hash = createHash('sha256')
|
|
35
|
+
.update(buffer)
|
|
36
|
+
.digest('hex')
|
|
37
|
+
.slice(0, 8);
|
|
38
|
+
const filename = `image-${hash}.${ext}`;
|
|
39
|
+
const relativePath = `${imagesDir}/${filename}`;
|
|
40
|
+
|
|
41
|
+
images.push({ filename, buffer });
|
|
42
|
+
|
|
43
|
+
return ``;
|
|
44
|
+
}
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
if (images.length > 0) {
|
|
48
|
+
fs.mkdirSync(imagesPath, { recursive: true });
|
|
49
|
+
for (const img of images) {
|
|
50
|
+
fs.writeFileSync(path.join(imagesPath, img.filename), img.buffer);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return { markdown: updatedMarkdown, extracted: images.length };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Extract base64 images from markdown into memory buffers and rewrite paths.
|
|
59
|
+
* Does NOT write to disk — intended for streaming into archives.
|
|
60
|
+
*
|
|
61
|
+
* @param {string} markdown - Markdown content with data URI images
|
|
62
|
+
* @param {string} [imagesDir='images'] - Directory prefix for image paths
|
|
63
|
+
* @returns {{markdown: string, images: Array<{filename: string, buffer: Buffer}>}}
|
|
64
|
+
*/
|
|
65
|
+
export function extractBase64ToBuffers(markdown, imagesDir = 'images') {
|
|
66
|
+
const images = [];
|
|
67
|
+
const updatedMarkdown = markdown.replace(
|
|
68
|
+
BASE64_IMAGE_REGEX,
|
|
69
|
+
(match, altText, mimeExt, base64Data) => {
|
|
70
|
+
const ext =
|
|
71
|
+
mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
|
|
72
|
+
|
|
73
|
+
let buffer;
|
|
74
|
+
try {
|
|
75
|
+
buffer = Buffer.from(base64Data, 'base64');
|
|
76
|
+
} catch {
|
|
77
|
+
return match;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const hash = createHash('sha256')
|
|
81
|
+
.update(buffer)
|
|
82
|
+
.digest('hex')
|
|
83
|
+
.slice(0, 8);
|
|
84
|
+
const filename = `image-${hash}.${ext}`;
|
|
85
|
+
const relativePath = `${imagesDir}/${filename}`;
|
|
86
|
+
|
|
87
|
+
images.push({ filename, buffer });
|
|
88
|
+
return ``;
|
|
89
|
+
}
|
|
90
|
+
);
|
|
91
|
+
return { markdown: updatedMarkdown, images };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Strip base64 data URI images from markdown, leaving only the alt text
|
|
96
|
+
* as a placeholder. Used when keepOriginalLinks is enabled — base64 images
|
|
97
|
+
* have no original URL to restore, so we remove the heavy data URI.
|
|
98
|
+
*
|
|
99
|
+
* @param {string} markdown - Markdown content with data URI images
|
|
100
|
+
* @returns {{markdown: string, stripped: number}}
|
|
101
|
+
*/
|
|
102
|
+
export function stripBase64Images(markdown) {
|
|
103
|
+
let stripped = 0;
|
|
104
|
+
const updatedMarkdown = markdown.replace(
|
|
105
|
+
BASE64_IMAGE_REGEX,
|
|
106
|
+
(_match, altText) => {
|
|
107
|
+
stripped++;
|
|
108
|
+
return altText ? `*[image: ${altText}]*` : '';
|
|
109
|
+
}
|
|
110
|
+
);
|
|
111
|
+
return { markdown: updatedMarkdown, stripped };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* @param {string} markdown - Markdown content
|
|
116
|
+
* @returns {boolean}
|
|
117
|
+
*/
|
|
118
|
+
export function hasBase64Images(markdown) {
|
|
119
|
+
return BASE64_IMAGE_REGEX.test(markdown);
|
|
120
|
+
}
|
package/src/gdocs.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
import fetch from 'node-fetch';
|
|
8
8
|
import he from 'he';
|
|
9
9
|
import archiver from 'archiver';
|
|
10
|
-
import { convertHtmlToMarkdown } from './lib.js';
|
|
10
|
+
import { convertHtmlToMarkdown, prettyPrintHtml } from './lib.js';
|
|
11
11
|
|
|
12
12
|
const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
|
|
13
13
|
|
|
@@ -194,8 +194,8 @@ export function extractBase64Images(html) {
|
|
|
194
194
|
*
|
|
195
195
|
* Fetches the document as HTML, extracts embedded base64 images,
|
|
196
196
|
* and bundles everything into a ZIP archive containing:
|
|
197
|
-
* -
|
|
198
|
-
* -
|
|
197
|
+
* - document.md (Markdown version)
|
|
198
|
+
* - document.html (HTML version with local image paths)
|
|
199
199
|
* - images/ (extracted images)
|
|
200
200
|
*
|
|
201
201
|
* @param {string} url - Google Docs URL
|
|
@@ -213,7 +213,7 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
|
|
|
213
213
|
const { html: localHtml, images } = extractBase64Images(result.content);
|
|
214
214
|
|
|
215
215
|
// Convert the localized HTML to Markdown
|
|
216
|
-
const markdown = convertHtmlToMarkdown(localHtml
|
|
216
|
+
const markdown = convertHtmlToMarkdown(localHtml);
|
|
217
217
|
|
|
218
218
|
return {
|
|
219
219
|
documentId: result.documentId,
|
|
@@ -286,8 +286,10 @@ async function sendGDocsArchive(res, url, apiToken) {
|
|
|
286
286
|
|
|
287
287
|
const archive = archiver('zip', { zlib: { level: 9 } });
|
|
288
288
|
archive.pipe(res);
|
|
289
|
-
archive.append(archiveResult.markdown, { name: '
|
|
290
|
-
archive.append(archiveResult.html, {
|
|
289
|
+
archive.append(archiveResult.markdown, { name: 'document.md' });
|
|
290
|
+
archive.append(prettyPrintHtml(archiveResult.html), {
|
|
291
|
+
name: 'document.html',
|
|
292
|
+
});
|
|
291
293
|
for (const img of archiveResult.images) {
|
|
292
294
|
archive.append(img.data, { name: `images/${img.filename}` });
|
|
293
295
|
}
|
package/src/lib.js
CHANGED
|
@@ -167,7 +167,8 @@ export function convertHtmlToMarkdown(html, baseUrl) {
|
|
|
167
167
|
});
|
|
168
168
|
turndown.use(turndownPluginGfm.gfm);
|
|
169
169
|
// Decode HTML entities to unicode after markdown conversion
|
|
170
|
-
|
|
170
|
+
// Preserve non-breaking spaces as entities for clear marking
|
|
171
|
+
return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
|
|
171
172
|
}
|
|
172
173
|
|
|
173
174
|
// Convert relative URLs to absolute URLs in HTML content
|
|
@@ -502,7 +503,8 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
|
|
|
502
503
|
turndown.use(turndownPluginGfm.gfm);
|
|
503
504
|
|
|
504
505
|
// Decode HTML entities to unicode after markdown conversion
|
|
505
|
-
|
|
506
|
+
// Normalize non-breaking spaces to regular spaces in Markdown text
|
|
507
|
+
let markdown = he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
|
|
506
508
|
|
|
507
509
|
// Apply post-processing
|
|
508
510
|
if (postProcess) {
|
|
@@ -553,3 +555,56 @@ export function ensureUtf8(html) {
|
|
|
553
555
|
}
|
|
554
556
|
return html;
|
|
555
557
|
}
|
|
558
|
+
|
|
559
|
+
const VOID_TAGS = new Set([
|
|
560
|
+
'area',
|
|
561
|
+
'base',
|
|
562
|
+
'br',
|
|
563
|
+
'col',
|
|
564
|
+
'embed',
|
|
565
|
+
'hr',
|
|
566
|
+
'img',
|
|
567
|
+
'input',
|
|
568
|
+
'link',
|
|
569
|
+
'meta',
|
|
570
|
+
'param',
|
|
571
|
+
'source',
|
|
572
|
+
'track',
|
|
573
|
+
'wbr',
|
|
574
|
+
]);
|
|
575
|
+
|
|
576
|
+
export function prettyPrintHtml(html) {
|
|
577
|
+
const tagRe = /(<\/?[a-zA-Z][^>]*?>)/g;
|
|
578
|
+
const parts = html.split(tagRe).filter(Boolean);
|
|
579
|
+
let indent = 0;
|
|
580
|
+
const indentStr = ' ';
|
|
581
|
+
const lines = [];
|
|
582
|
+
|
|
583
|
+
for (const part of parts) {
|
|
584
|
+
const isTag = part.startsWith('<');
|
|
585
|
+
if (!isTag) {
|
|
586
|
+
const text = part.trim();
|
|
587
|
+
if (text) {
|
|
588
|
+
lines.push(indentStr.repeat(indent) + text);
|
|
589
|
+
}
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
const isClosing = part.startsWith('</');
|
|
593
|
+
const tagMatch = part.match(/^<\/?([a-zA-Z][a-zA-Z0-9]*)/);
|
|
594
|
+
const tagName = tagMatch ? tagMatch[1].toLowerCase() : '';
|
|
595
|
+
const isVoid = VOID_TAGS.has(tagName);
|
|
596
|
+
const isSelfClosing = part.endsWith('/>');
|
|
597
|
+
|
|
598
|
+
if (isClosing) {
|
|
599
|
+
indent = Math.max(0, indent - 1);
|
|
600
|
+
lines.push(indentStr.repeat(indent) + part);
|
|
601
|
+
} else if (isVoid || isSelfClosing) {
|
|
602
|
+
lines.push(indentStr.repeat(indent) + part);
|
|
603
|
+
} else {
|
|
604
|
+
lines.push(indentStr.repeat(indent) + part);
|
|
605
|
+
indent++;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
return `${lines.join('\n')}\n`;
|
|
610
|
+
}
|
package/src/markdown.js
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { fetchHtml, convertHtmlToMarkdown } from './lib.js';
|
|
2
|
+
import { stripBase64Images } from './extract-images.js';
|
|
2
3
|
|
|
3
4
|
export async function markdownHandler(req, res) {
|
|
4
5
|
const url = req.query.url;
|
|
5
6
|
if (!url) {
|
|
6
7
|
return res.status(400).send('Missing `url` parameter');
|
|
7
8
|
}
|
|
9
|
+
const embedImages = req.query.embedImages === 'true';
|
|
8
10
|
try {
|
|
9
11
|
const html = await fetchHtml(url);
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
let markdown = convertHtmlToMarkdown(html, url);
|
|
13
|
+
if (!embedImages) {
|
|
14
|
+
const strip = stripBase64Images(markdown);
|
|
15
|
+
markdown = strip.markdown;
|
|
16
|
+
}
|
|
12
17
|
res.type('text/markdown').send(markdown);
|
|
13
18
|
} catch (err) {
|
|
14
19
|
console.error(err);
|
package/src/postprocess.js
CHANGED
|
@@ -71,10 +71,8 @@ export function postProcessMarkdown(markdown, options = {}) {
|
|
|
71
71
|
export function applyUnicodeNormalization(text) {
|
|
72
72
|
let result = text;
|
|
73
73
|
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
// for inline math $...$ delimiters.
|
|
77
|
-
result = result.replace(/\u00A0/g, ' ');
|
|
74
|
+
// Preserve non-breaking spaces as entities for clear marking
|
|
75
|
+
result = result.replace(/\u00A0/g, ' ');
|
|
78
76
|
|
|
79
77
|
// Normalize curly quotes to straight quotes
|
|
80
78
|
result = result.replace(/[\u2018\u2019]/g, "'");
|
package/tests/unit/batch.test.js
CHANGED
|
@@ -22,7 +22,7 @@ describe('batch module', () => {
|
|
|
22
22
|
},
|
|
23
23
|
defaults: {
|
|
24
24
|
archivePath: 'archive',
|
|
25
|
-
markdownFile: '
|
|
25
|
+
markdownFile: 'document.md',
|
|
26
26
|
imagesDir: 'images',
|
|
27
27
|
hasLocalImages: true,
|
|
28
28
|
},
|
|
@@ -37,7 +37,7 @@ describe('batch module', () => {
|
|
|
37
37
|
|
|
38
38
|
it('merges defaults into article config', () => {
|
|
39
39
|
const article = getArticle(sampleConfig, '0.0.0');
|
|
40
|
-
expect(article.markdownFile).toBe('
|
|
40
|
+
expect(article.markdownFile).toBe('document.md');
|
|
41
41
|
expect(article.hasLocalImages).toBe(true);
|
|
42
42
|
});
|
|
43
43
|
|
|
@@ -59,7 +59,7 @@ describe('batch module', () => {
|
|
|
59
59
|
it('returns all articles with defaults merged', () => {
|
|
60
60
|
const articles = getAllArticles(sampleConfig);
|
|
61
61
|
expect(articles).toHaveLength(2);
|
|
62
|
-
expect(articles[0].markdownFile).toBe('
|
|
62
|
+
expect(articles[0].markdownFile).toBe('document.md');
|
|
63
63
|
expect(articles[1].language).toBe('ru');
|
|
64
64
|
});
|
|
65
65
|
});
|
|
@@ -78,7 +78,7 @@ describe('batch module', () => {
|
|
|
78
78
|
it('sets default archive paths', () => {
|
|
79
79
|
const config = createConfigFromUrls(['https://habr.com/article/123']);
|
|
80
80
|
expect(config.articles['1'].archivePath).toContain('habr-com');
|
|
81
|
-
expect(config.articles['1'].markdownFile).toBe('
|
|
81
|
+
expect(config.articles['1'].markdownFile).toBe('document.md');
|
|
82
82
|
});
|
|
83
83
|
});
|
|
84
84
|
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
import {
|
|
6
|
+
extractAndSaveImages,
|
|
7
|
+
extractBase64ToBuffers,
|
|
8
|
+
stripBase64Images,
|
|
9
|
+
hasBase64Images,
|
|
10
|
+
} from '../../src/extract-images.js';
|
|
11
|
+
|
|
12
|
+
describe('extract-images module', () => {
|
|
13
|
+
let tmpDir;
|
|
14
|
+
|
|
15
|
+
beforeEach(() => {
|
|
16
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'extract-images-test-'));
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
afterEach(() => {
|
|
20
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
// 1x1 red PNG pixel as base64
|
|
24
|
+
const TINY_PNG =
|
|
25
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
|
|
26
|
+
// 1x1 white JPEG as base64
|
|
27
|
+
const TINY_JPEG =
|
|
28
|
+
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP///wAAAf/bAEMA/9sAQwD/2wBDAf///wAAAf/bAEMA/9sAQwD/2Q==';
|
|
29
|
+
|
|
30
|
+
function contentHash(base64Data) {
|
|
31
|
+
const buf = Buffer.from(base64Data, 'base64');
|
|
32
|
+
return createHash('sha256').update(buf).digest('hex').slice(0, 8);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const PNG_HASH = contentHash(TINY_PNG);
|
|
36
|
+
const JPEG_HASH = contentHash(TINY_JPEG);
|
|
37
|
+
|
|
38
|
+
describe('extractAndSaveImages', () => {
|
|
39
|
+
it('extracts a single PNG image from markdown', () => {
|
|
40
|
+
const md = `# Hello\n\n\n\nEnd.`;
|
|
41
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
42
|
+
|
|
43
|
+
expect(result.extracted).toBe(1);
|
|
44
|
+
expect(result.markdown).toContain(
|
|
45
|
+
``
|
|
46
|
+
);
|
|
47
|
+
expect(result.markdown).not.toContain('data:image');
|
|
48
|
+
|
|
49
|
+
const imgPath = path.join(tmpDir, 'images', `image-${PNG_HASH}.png`);
|
|
50
|
+
expect(fs.existsSync(imgPath)).toBe(true);
|
|
51
|
+
const buf = fs.readFileSync(imgPath);
|
|
52
|
+
expect(buf.length).toBeGreaterThan(0);
|
|
53
|
+
// Verify PNG magic bytes
|
|
54
|
+
expect(buf[0]).toBe(0x89);
|
|
55
|
+
expect(buf[1]).toBe(0x50); // P
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('extracts multiple images with content-hash filenames', () => {
|
|
59
|
+
const md = [
|
|
60
|
+
``,
|
|
61
|
+
``,
|
|
62
|
+
].join('\n');
|
|
63
|
+
|
|
64
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
65
|
+
expect(result.extracted).toBe(2);
|
|
66
|
+
expect(result.markdown).toContain(`images/image-${PNG_HASH}.png`);
|
|
67
|
+
expect(result.markdown).toContain(`images/image-${JPEG_HASH}.jpg`);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('produces stable filenames for duplicate images', () => {
|
|
71
|
+
const md = [
|
|
72
|
+
``,
|
|
73
|
+
``,
|
|
74
|
+
].join('\n');
|
|
75
|
+
|
|
76
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
77
|
+
expect(result.extracted).toBe(2);
|
|
78
|
+
// Both should produce the same hash-based filename
|
|
79
|
+
const matches = result.markdown.match(
|
|
80
|
+
new RegExp(`image-${PNG_HASH}\\.png`, 'g')
|
|
81
|
+
);
|
|
82
|
+
expect(matches).toHaveLength(2);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('handles JPEG mime type correctly', () => {
|
|
86
|
+
const md = ``;
|
|
87
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
88
|
+
|
|
89
|
+
expect(result.extracted).toBe(1);
|
|
90
|
+
expect(result.markdown).toContain(`images/image-${JPEG_HASH}.jpg`);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it('uses custom imagesDir', () => {
|
|
94
|
+
const md = ``;
|
|
95
|
+
const result = extractAndSaveImages(md, tmpDir, {
|
|
96
|
+
imagesDir: 'my-images',
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
expect(result.extracted).toBe(1);
|
|
100
|
+
expect(result.markdown).toContain(`my-images/image-${PNG_HASH}.png`);
|
|
101
|
+
expect(
|
|
102
|
+
fs.existsSync(path.join(tmpDir, 'my-images', `image-${PNG_HASH}.png`))
|
|
103
|
+
).toBe(true);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('returns markdown unchanged when no base64 images', () => {
|
|
107
|
+
const md = '# No images\n\nJust text.';
|
|
108
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
109
|
+
|
|
110
|
+
expect(result.extracted).toBe(0);
|
|
111
|
+
expect(result.markdown).toBe(md);
|
|
112
|
+
expect(fs.existsSync(path.join(tmpDir, 'images'))).toBe(false);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('preserves non-base64 image references', () => {
|
|
116
|
+
const md =
|
|
117
|
+
'\n';
|
|
118
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
119
|
+
|
|
120
|
+
expect(result.extracted).toBe(0);
|
|
121
|
+
expect(result.markdown).toBe(md);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('preserves alt text during extraction', () => {
|
|
125
|
+
const md = ``;
|
|
126
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
127
|
+
|
|
128
|
+
expect(result.markdown).toContain(
|
|
129
|
+
``
|
|
130
|
+
);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it('handles SVG data URIs', () => {
|
|
134
|
+
const svgContent =
|
|
135
|
+
'<svg xmlns="http://www.w3.org/2000/svg" width="1" height="1"><rect fill="red" width="1" height="1"/></svg>';
|
|
136
|
+
const svgBase64 = Buffer.from(svgContent).toString('base64');
|
|
137
|
+
const svgHash = contentHash(svgBase64);
|
|
138
|
+
const md = ``;
|
|
139
|
+
const result = extractAndSaveImages(md, tmpDir);
|
|
140
|
+
|
|
141
|
+
expect(result.extracted).toBe(1);
|
|
142
|
+
expect(result.markdown).toContain(`images/image-${svgHash}.svg`);
|
|
143
|
+
|
|
144
|
+
const imgPath = path.join(tmpDir, 'images', `image-${svgHash}.svg`);
|
|
145
|
+
expect(fs.existsSync(imgPath)).toBe(true);
|
|
146
|
+
const content = fs.readFileSync(imgPath, 'utf-8');
|
|
147
|
+
expect(content).toContain('<svg');
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
describe('extractBase64ToBuffers', () => {
|
|
152
|
+
it('extracts base64 images to in-memory buffers', () => {
|
|
153
|
+
const md = `# Hello\n\n\n\nEnd.`;
|
|
154
|
+
const result = extractBase64ToBuffers(md);
|
|
155
|
+
|
|
156
|
+
expect(result.images).toHaveLength(1);
|
|
157
|
+
expect(result.images[0].filename).toMatch(/^image-[0-9a-f]{8}\.png$/);
|
|
158
|
+
expect(result.images[0].buffer.length).toBeGreaterThan(0);
|
|
159
|
+
expect(result.markdown).toContain(`images/${result.images[0].filename}`);
|
|
160
|
+
expect(result.markdown).not.toContain('data:image');
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('uses custom imagesDir prefix', () => {
|
|
164
|
+
const md = ``;
|
|
165
|
+
const result = extractBase64ToBuffers(md, 'assets');
|
|
166
|
+
|
|
167
|
+
expect(result.markdown).toContain('assets/image-');
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
it('does not write files to disk', () => {
|
|
171
|
+
const md = ``;
|
|
172
|
+
extractBase64ToBuffers(md);
|
|
173
|
+
expect(fs.existsSync(path.join(tmpDir, 'images'))).toBe(false);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
describe('stripBase64Images', () => {
|
|
178
|
+
it('strips base64 images and leaves alt text placeholder', () => {
|
|
179
|
+
const md = ``;
|
|
180
|
+
const result = stripBase64Images(md);
|
|
181
|
+
|
|
182
|
+
expect(result.stripped).toBe(1);
|
|
183
|
+
expect(result.markdown).toBe('*[image: my image]*');
|
|
184
|
+
expect(result.markdown).not.toContain('data:image');
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('strips base64 images with empty alt text', () => {
|
|
188
|
+
const md = ``;
|
|
189
|
+
const result = stripBase64Images(md);
|
|
190
|
+
|
|
191
|
+
expect(result.stripped).toBe(1);
|
|
192
|
+
expect(result.markdown).toBe('');
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('preserves remote image URLs', () => {
|
|
196
|
+
const md = '';
|
|
197
|
+
const result = stripBase64Images(md);
|
|
198
|
+
|
|
199
|
+
expect(result.stripped).toBe(0);
|
|
200
|
+
expect(result.markdown).toBe(md);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it('strips multiple base64 images', () => {
|
|
204
|
+
const md = [
|
|
205
|
+
``,
|
|
206
|
+
``,
|
|
207
|
+
'',
|
|
208
|
+
].join('\n');
|
|
209
|
+
const result = stripBase64Images(md);
|
|
210
|
+
|
|
211
|
+
expect(result.stripped).toBe(2);
|
|
212
|
+
expect(result.markdown).toContain('*[image: a]*');
|
|
213
|
+
expect(result.markdown).toContain('*[image: b]*');
|
|
214
|
+
expect(result.markdown).toContain('https://example.com/img.png');
|
|
215
|
+
});
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
describe('hasBase64Images', () => {
|
|
219
|
+
it('returns true when markdown has base64 images', () => {
|
|
220
|
+
expect(hasBase64Images(``)).toBe(
|
|
221
|
+
true
|
|
222
|
+
);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('returns false when no base64 images', () => {
|
|
226
|
+
expect(hasBase64Images('')).toBe(false);
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
it('returns false for empty string', () => {
|
|
230
|
+
expect(hasBase64Images('')).toBe(false);
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
});
|
package/tests/unit/gdocs.test.js
CHANGED
|
@@ -6,6 +6,14 @@ import {
|
|
|
6
6
|
extractBase64Images,
|
|
7
7
|
GDOCS_EXPORT_FORMATS,
|
|
8
8
|
} from '../../src/gdocs.js';
|
|
9
|
+
import {
|
|
10
|
+
extractAndSaveImages,
|
|
11
|
+
extractBase64ToBuffers,
|
|
12
|
+
stripBase64Images,
|
|
13
|
+
} from '../../src/extract-images.js';
|
|
14
|
+
import fs from 'fs';
|
|
15
|
+
import os from 'os';
|
|
16
|
+
import path from 'path';
|
|
9
17
|
|
|
10
18
|
jest.setTimeout(30000);
|
|
11
19
|
|
|
@@ -200,4 +208,71 @@ describe('gdocs', () => {
|
|
|
200
208
|
expect(updated).toContain('images/image-01.gif');
|
|
201
209
|
});
|
|
202
210
|
});
|
|
211
|
+
|
|
212
|
+
describe('image extraction pipeline (issue #53)', () => {
|
|
213
|
+
const TINY_PNG =
|
|
214
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
|
|
215
|
+
let tmpDir;
|
|
216
|
+
|
|
217
|
+
beforeEach(() => {
|
|
218
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gdocs-issue53-'));
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
afterEach(() => {
|
|
222
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it('extractBase64Images from HTML produces images with data buffers', () => {
|
|
226
|
+
const html = `<html><body><img src="data:image/png;base64,${TINY_PNG}" alt="photo"><p>Hello</p></body></html>`;
|
|
227
|
+
const { html: localHtml, images } = extractBase64Images(html);
|
|
228
|
+
|
|
229
|
+
expect(images).toHaveLength(1);
|
|
230
|
+
expect(images[0].filename).toBe('image-01.png');
|
|
231
|
+
expect(images[0].data.length).toBeGreaterThan(0);
|
|
232
|
+
// Verify PNG magic bytes in extracted data
|
|
233
|
+
expect(images[0].data[0]).toBe(0x89);
|
|
234
|
+
expect(images[0].data[1]).toBe(0x50); // P
|
|
235
|
+
expect(localHtml).toContain('images/image-01.png');
|
|
236
|
+
expect(localHtml).not.toContain('data:image');
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it('default markdown mode extracts base64 images to disk (not strip)', () => {
|
|
240
|
+
const markdown = `# Hello\n\n\n\nEnd.`;
|
|
241
|
+
const result = extractAndSaveImages(markdown, tmpDir);
|
|
242
|
+
|
|
243
|
+
expect(result.extracted).toBe(1);
|
|
244
|
+
expect(result.markdown).not.toContain('data:image');
|
|
245
|
+
expect(result.markdown).toContain('images/image-');
|
|
246
|
+
expect(result.markdown).toContain('.png');
|
|
247
|
+
|
|
248
|
+
const imagesDir = path.join(tmpDir, 'images');
|
|
249
|
+
expect(fs.existsSync(imagesDir)).toBe(true);
|
|
250
|
+
const files = fs.readdirSync(imagesDir);
|
|
251
|
+
expect(files).toHaveLength(1);
|
|
252
|
+
expect(files[0]).toMatch(/^image-.*\.png$/);
|
|
253
|
+
|
|
254
|
+
const imgBuf = fs.readFileSync(path.join(imagesDir, files[0]));
|
|
255
|
+
expect(imgBuf[0]).toBe(0x89); // PNG magic
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it('keepOriginalLinks strips base64 images', () => {
|
|
259
|
+
const markdown = ``;
|
|
260
|
+
const result = stripBase64Images(markdown);
|
|
261
|
+
|
|
262
|
+
expect(result.stripped).toBe(1);
|
|
263
|
+
expect(result.markdown).toBe('*[image: photo]*');
|
|
264
|
+
expect(result.markdown).not.toContain('data:image');
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
it('extractBase64ToBuffers produces in-memory buffers for archive', () => {
|
|
268
|
+
const markdown = ``;
|
|
269
|
+
const result = extractBase64ToBuffers(markdown);
|
|
270
|
+
|
|
271
|
+
expect(result.images).toHaveLength(1);
|
|
272
|
+
expect(result.images[0].buffer.length).toBeGreaterThan(0);
|
|
273
|
+
expect(result.images[0].buffer[0]).toBe(0x89); // PNG magic
|
|
274
|
+
expect(result.markdown).toContain('images/');
|
|
275
|
+
expect(result.markdown).not.toContain('data:image');
|
|
276
|
+
});
|
|
277
|
+
});
|
|
203
278
|
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { convertHtmlToMarkdown } from '../../src/lib.js';
|
|
2
|
+
import { extractBase64Images } from '../../src/gdocs.js';
|
|
3
|
+
|
|
4
|
+
const PNG_B64 =
|
|
5
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=';
|
|
6
|
+
|
|
7
|
+
const IMG_REF_PATTERN = /!\[[^\]]*\]\([^)]*images\/[^)]+\)/g;
|
|
8
|
+
|
|
9
|
+
describe('heading image parity', () => {
|
|
10
|
+
it('<img> inside <h1> survives Turndown and stays in sync with extraction', () => {
|
|
11
|
+
const html = `<html><body><h1>Title <img src="data:image/png;base64,${PNG_B64}" alt="x"></h1><p>Body</p></body></html>`;
|
|
12
|
+
|
|
13
|
+
const { html: localHtml, images } = extractBase64Images(html);
|
|
14
|
+
const md = convertHtmlToMarkdown(localHtml);
|
|
15
|
+
const mdImgCount = (md.match(IMG_REF_PATTERN) || []).length;
|
|
16
|
+
|
|
17
|
+
expect(images).toHaveLength(1);
|
|
18
|
+
expect(mdImgCount).toBe(images.length);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('<img> inside all heading levels is preserved', () => {
|
|
22
|
+
for (let level = 1; level <= 6; level++) {
|
|
23
|
+
const tag = `h${level}`;
|
|
24
|
+
const html = `<html><body><${tag}><img src="data:image/png;base64,${PNG_B64}" alt="icon"> Heading</${tag}></body></html>`;
|
|
25
|
+
|
|
26
|
+
const { html: localHtml, images } = extractBase64Images(html);
|
|
27
|
+
const md = convertHtmlToMarkdown(localHtml);
|
|
28
|
+
const refs = (md.match(IMG_REF_PATTERN) || []).length;
|
|
29
|
+
|
|
30
|
+
expect(refs).toBe(images.length);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('image count parity across full pipeline', () => {
|
|
35
|
+
const html = `<html><body>
|
|
36
|
+
<h1><img src="data:image/png;base64,${PNG_B64}" alt="h1"> Chapter</h1>
|
|
37
|
+
<p>Text <img src="data:image/png;base64,${PNG_B64}" alt="p1"> here.</p>
|
|
38
|
+
<h2><img src="data:image/png;base64,${PNG_B64}" alt="h2"> Section</h2>
|
|
39
|
+
<p><img src="data:image/png;base64,${PNG_B64}" alt="p2"></p>
|
|
40
|
+
</body></html>`;
|
|
41
|
+
|
|
42
|
+
const { html: localHtml, images } = extractBase64Images(html);
|
|
43
|
+
const md = convertHtmlToMarkdown(localHtml);
|
|
44
|
+
const mdRefCount = (md.match(IMG_REF_PATTERN) || []).length;
|
|
45
|
+
|
|
46
|
+
expect(images).toHaveLength(4);
|
|
47
|
+
expect(mdRefCount).toBe(images.length);
|
|
48
|
+
});
|
|
49
|
+
});
|