@link-assistant/web-capture 1.10.10 → 1.10.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/bin/web-capture.js +7 -15
- package/package.json +2 -2
- package/src/archive.js +3 -3
- package/src/markdown.js +2 -2
- package/tests/unit/package-runtime-dependencies.test.js +26 -0
package/CHANGELOG.md
CHANGED
package/bin/web-capture.js
CHANGED
|
@@ -549,15 +549,14 @@ async function writeGoogleDocsArchive({
|
|
|
549
549
|
archiveFormat,
|
|
550
550
|
prettyHtml,
|
|
551
551
|
}) {
|
|
552
|
-
const {
|
|
552
|
+
const { ZipArchive } = await import('archiver');
|
|
553
553
|
const { prettyPrintHtml } = await import('../src/lib.js');
|
|
554
554
|
const output =
|
|
555
555
|
explicitOutput === '-'
|
|
556
556
|
? null
|
|
557
557
|
: explicitOutput ||
|
|
558
558
|
deriveOutputPath(absoluteUrl, archiveExtension(archiveFormat), dataDir);
|
|
559
|
-
const
|
|
560
|
-
const archive = archiverFn('zip', { zlib: { level: 9 } });
|
|
559
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
561
560
|
archive.append(archiveResult.markdown, { name: 'document.md' });
|
|
562
561
|
archive.append(
|
|
563
562
|
prettyHtml ? prettyPrintHtml(archiveResult.html) : archiveResult.html,
|
|
@@ -835,7 +834,7 @@ async function captureUrl(url, options) {
|
|
|
835
834
|
try {
|
|
836
835
|
if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
|
|
837
836
|
const { fetchGoogleDocAsArchive } = await import('../src/gdocs.js');
|
|
838
|
-
const {
|
|
837
|
+
const { ZipArchive } = await import('archiver');
|
|
839
838
|
const archiveFormat = options.archiveFormat || 'zip';
|
|
840
839
|
const ext =
|
|
841
840
|
archiveFormat === 'tar.gz' || archiveFormat === 'gz'
|
|
@@ -855,9 +854,7 @@ async function captureUrl(url, options) {
|
|
|
855
854
|
: explicitOutput || deriveOutputPath(absoluteUrl, ext, dataDir);
|
|
856
855
|
if (output) {
|
|
857
856
|
const outStream = fs.createWriteStream(output);
|
|
858
|
-
const archive =
|
|
859
|
-
? archiver.default('zip', { zlib: { level: 9 } })
|
|
860
|
-
: archiver('zip', { zlib: { level: 9 } });
|
|
857
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
861
858
|
archive.pipe(outStream);
|
|
862
859
|
const { prettyPrintHtml: ppHtml } = await import('../src/lib.js');
|
|
863
860
|
archive.append(archiveResult.markdown, { name: 'document.md' });
|
|
@@ -876,10 +873,7 @@ async function captureUrl(url, options) {
|
|
|
876
873
|
} else {
|
|
877
874
|
const { PassThrough } = await import('stream');
|
|
878
875
|
const passthrough = new PassThrough();
|
|
879
|
-
const
|
|
880
|
-
const archiverFn =
|
|
881
|
-
archiverMod.default?.default || archiverMod.default;
|
|
882
|
-
const archive = archiverFn('zip', { zlib: { level: 9 } });
|
|
876
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
883
877
|
archive.pipe(passthrough);
|
|
884
878
|
passthrough.pipe(process.stdout);
|
|
885
879
|
const { prettyPrintHtml: ppHtml2 } = await import('../src/lib.js');
|
|
@@ -1111,7 +1105,7 @@ async function captureUrl(url, options) {
|
|
|
1111
1105
|
console.error(`DOCX saved to: ${outPath}`);
|
|
1112
1106
|
} else if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
|
|
1113
1107
|
// ZIP archive
|
|
1114
|
-
const {
|
|
1108
|
+
const { ZipArchive } = await import('archiver');
|
|
1115
1109
|
const { fetchHtml, convertHtmlToMarkdown, convertRelativeUrls } =
|
|
1116
1110
|
await import('../src/lib.js');
|
|
1117
1111
|
const { retry } = await import('../src/retry.js');
|
|
@@ -1131,9 +1125,7 @@ async function captureUrl(url, options) {
|
|
|
1131
1125
|
explicitOutput ||
|
|
1132
1126
|
`${new URL(absoluteUrl).hostname.replace(/\./g, '-')}-archive.zip`;
|
|
1133
1127
|
const outStream = fs.createWriteStream(outPath);
|
|
1134
|
-
const archive =
|
|
1135
|
-
? archiver.default('zip', { zlib: { level: 9 } })
|
|
1136
|
-
: archiver('zip', { zlib: { level: 9 } });
|
|
1128
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
1137
1129
|
archive.pipe(outStream);
|
|
1138
1130
|
|
|
1139
1131
|
// Collect images
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-assistant/web-capture",
|
|
3
|
-
"version": "1.10.
|
|
3
|
+
"version": "1.10.11",
|
|
4
4
|
"description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@kreuzberg/html-to-markdown-node": "^3.5.5",
|
|
36
|
-
"archiver": "^
|
|
36
|
+
"archiver": "^8.0.0",
|
|
37
37
|
"browser-commander": "^0.8.0",
|
|
38
38
|
"cheerio": "^1.0.0",
|
|
39
39
|
"docx": "^9.6.1",
|
package/src/archive.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* documentFormat - 'markdown' (default) or 'html' - format of the main document
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import
|
|
18
|
+
import { ZipArchive } from 'archiver';
|
|
19
19
|
import fetch from 'node-fetch';
|
|
20
20
|
import * as cheerio from 'cheerio';
|
|
21
21
|
import { URL } from 'url';
|
|
@@ -88,7 +88,7 @@ export async function archiveHandler(req, res) {
|
|
|
88
88
|
`attachment; filename="${hostname}-archive.zip"`
|
|
89
89
|
);
|
|
90
90
|
|
|
91
|
-
const archive =
|
|
91
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
92
92
|
archive.pipe(res);
|
|
93
93
|
|
|
94
94
|
// Build the image map for local downloads
|
|
@@ -251,7 +251,7 @@ function appendMarkdownAndImages(archive, markdown) {
|
|
|
251
251
|
* @returns {Promise<Buffer>} the finalized ZIP archive bytes
|
|
252
252
|
*/
|
|
253
253
|
export async function buildArchiveFromHtml(html, baseUrl) {
|
|
254
|
-
const archive =
|
|
254
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
255
255
|
const chunks = [];
|
|
256
256
|
archive.on('data', (chunk) => chunks.push(chunk));
|
|
257
257
|
const finished = new Promise((resolve, reject) => {
|
package/src/markdown.js
CHANGED
|
@@ -17,7 +17,7 @@ import { convertWithKreuzberg, isKreuzbergAvailable } from './kreuzberg.js';
|
|
|
17
17
|
import { applyImageMode } from './extract-images.js';
|
|
18
18
|
import { createBrowser, getBrowserEngine } from './browser.js';
|
|
19
19
|
import { retry } from './retry.js';
|
|
20
|
-
import
|
|
20
|
+
import { ZipArchive } from 'archiver';
|
|
21
21
|
import fetch from 'node-fetch';
|
|
22
22
|
import * as cheerio from 'cheerio';
|
|
23
23
|
|
|
@@ -300,7 +300,7 @@ async function sendTextPasteMarkdownArchive(res, url, markdown, rawText) {
|
|
|
300
300
|
res.setHeader('Content-Type', 'application/zip');
|
|
301
301
|
res.setHeader('Content-Disposition', `attachment; filename="${pasteId}.zip"`);
|
|
302
302
|
|
|
303
|
-
const archive =
|
|
303
|
+
const archive = new ZipArchive({ zlib: { level: 9 } });
|
|
304
304
|
archive.pipe(res);
|
|
305
305
|
archive.append(
|
|
306
306
|
[
|
|
@@ -10,6 +10,25 @@ function readJson(relativePath) {
|
|
|
10
10
|
return JSON.parse(readFileSync(resolve(jsRoot, relativePath), 'utf8'));
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
function collectProductionDependencies(packageLock, packageName) {
|
|
14
|
+
const visited = new Set();
|
|
15
|
+
|
|
16
|
+
function visit(name) {
|
|
17
|
+
if (visited.has(name)) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
visited.add(name);
|
|
21
|
+
|
|
22
|
+
const entry = packageLock.packages[`node_modules/${name}`];
|
|
23
|
+
for (const dependency of Object.keys(entry?.dependencies || {})) {
|
|
24
|
+
visit(dependency);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
visit(packageName);
|
|
29
|
+
return visited;
|
|
30
|
+
}
|
|
31
|
+
|
|
13
32
|
const runtimeImports = [
|
|
14
33
|
{
|
|
15
34
|
packageName: 'turndown-plugin-gfm',
|
|
@@ -18,6 +37,13 @@ const runtimeImports = [
|
|
|
18
37
|
];
|
|
19
38
|
|
|
20
39
|
describe('runtime package dependencies', () => {
|
|
40
|
+
test('archiver production dependencies do not include deprecated glob', () => {
|
|
41
|
+
const packageLock = readJson('package-lock.json');
|
|
42
|
+
const dependencies = collectProductionDependencies(packageLock, 'archiver');
|
|
43
|
+
|
|
44
|
+
expect([...dependencies]).not.toContain('glob');
|
|
45
|
+
});
|
|
46
|
+
|
|
21
47
|
test.each(runtimeImports)(
|
|
22
48
|
'$packageName is declared as a production dependency',
|
|
23
49
|
({ packageName, sourceFile }) => {
|