@statewalker/content-extractors 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -10
- package/dist/collect-bytes.d.ts +10 -2
- package/dist/collect-bytes.d.ts.map +1 -1
- package/dist/collect-bytes.js +6 -4
- package/dist/collect-bytes.js.map +1 -1
- package/dist/extractor-registry.d.ts +2 -1
- package/dist/extractor-registry.d.ts.map +1 -1
- package/dist/extractor-registry.js +3 -2
- package/dist/extractor-registry.js.map +1 -1
- package/dist/extractors/docx-extractor.d.ts +5 -1
- package/dist/extractors/docx-extractor.d.ts.map +1 -1
- package/dist/extractors/docx-extractor.js +20 -11
- package/dist/extractors/docx-extractor.js.map +1 -1
- package/dist/extractors/html-extractor.d.ts +3 -0
- package/dist/extractors/html-extractor.d.ts.map +1 -1
- package/dist/extractors/html-extractor.js +8 -5
- package/dist/extractors/html-extractor.js.map +1 -1
- package/dist/extractors/index.d.ts +12 -9
- package/dist/extractors/index.d.ts.map +1 -1
- package/dist/extractors/index.js +23 -18
- package/dist/extractors/index.js.map +1 -1
- package/dist/extractors/markdown-extractor.d.ts +3 -0
- package/dist/extractors/markdown-extractor.d.ts.map +1 -1
- package/dist/extractors/markdown-extractor.js +8 -4
- package/dist/extractors/markdown-extractor.js.map +1 -1
- package/dist/extractors/pdf-extractor.d.ts +3 -0
- package/dist/extractors/pdf-extractor.d.ts.map +1 -1
- package/dist/extractors/pdf-extractor.js +17 -11
- package/dist/extractors/pdf-extractor.js.map +1 -1
- package/dist/extractors/text-extractor.d.ts +3 -0
- package/dist/extractors/text-extractor.d.ts.map +1 -1
- package/dist/extractors/text-extractor.js +8 -4
- package/dist/extractors/text-extractor.js.map +1 -1
- package/dist/extractors/xlsx-extractor.d.ts +5 -1
- package/dist/extractors/xlsx-extractor.d.ts.map +1 -1
- package/dist/extractors/xlsx-extractor.js +56 -36
- package/dist/extractors/xlsx-extractor.js.map +1 -1
- package/dist/html-to-markdown.d.ts +3 -1
- package/dist/html-to-markdown.d.ts.map +1 -1
- package/dist/html-to-markdown.js +9 -1
- package/dist/html-to-markdown.js.map +1 -1
- package/dist/index.d.ts +11 -9
- package/dist/index.js +9 -7
- package/dist/limits.d.ts +20 -0
- package/dist/limits.d.ts.map +1 -0
- package/dist/limits.js +9 -0
- package/dist/limits.js.map +1 -0
- package/dist/markdown-to-html.d.ts +11 -3
- package/dist/markdown-to-html.d.ts.map +1 -1
- package/dist/markdown-to-html.js +13 -6
- package/dist/markdown-to-html.js.map +1 -1
- package/dist/mime-utils.js +3 -2
- package/dist/mime-utils.js.map +1 -1
- package/dist/zip-size.d.ts +16 -0
- package/dist/zip-size.d.ts.map +1 -0
- package/dist/zip-size.js +90 -0
- package/dist/zip-size.js.map +1 -0
- package/package.json +8 -4
- package/src/collect-bytes.ts +17 -2
- package/src/extractor-registry.ts +3 -2
- package/src/extractors/docx-extractor.ts +26 -11
- package/src/extractors/html-extractor.ts +8 -4
- package/src/extractors/index.ts +29 -23
- package/src/extractors/markdown-extractor.ts +8 -3
- package/src/extractors/pdf-extractor.ts +20 -11
- package/src/extractors/text-extractor.ts +8 -3
- package/src/extractors/xlsx-extractor.ts +72 -41
- package/src/html-to-markdown.ts +4 -1
- package/src/index.ts +2 -0
- package/src/limits.ts +20 -0
- package/src/markdown-to-html.ts +16 -10
- package/src/mime-utils.ts +3 -2
- package/src/zip-size.ts +113 -0
package/src/zip-size.ts
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
const EOCD_SIG = 0x06054b50;
|
|
2
|
+
const ZIP64_LOCATOR_SIG = 0x07064b50;
|
|
3
|
+
const ZIP64_EOCD_SIG = 0x06064b50;
|
|
4
|
+
const CD_ENTRY_SIG = 0x02014b50;
|
|
5
|
+
const EOCD_SIZE = 22;
|
|
6
|
+
const MAX_COMMENT = 0xffff;
|
|
7
|
+
|
|
8
|
+
function fail(message: string): never {
|
|
9
|
+
throw new Error(`Invalid ZIP archive: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Sums the uncompressed sizes of all entries declared in a ZIP archive's central directory,
|
|
14
|
+
* without inflating anything. Supports ZIP64. Throws when the bytes are not a readable ZIP.
|
|
15
|
+
*
|
|
16
|
+
* The sizes are what the archive claims; a parser that inflates the data still has to check
|
|
17
|
+
* them, which JSZip (used by mammoth and exceljs) does.
|
|
18
|
+
*/
|
|
19
|
+
export function zipUncompressedSize(bytes: Uint8Array): number {
|
|
20
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
21
|
+
const len = bytes.byteLength;
|
|
22
|
+
|
|
23
|
+
// The end-of-central-directory record sits at the end, followed by a comment of up to 64 KiB.
|
|
24
|
+
let eocd = -1;
|
|
25
|
+
const stop = Math.max(0, len - EOCD_SIZE - MAX_COMMENT);
|
|
26
|
+
for (let i = len - EOCD_SIZE; i >= stop; i--) {
|
|
27
|
+
if (view.getUint32(i, true) === EOCD_SIG) {
|
|
28
|
+
eocd = i;
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (eocd < 0) fail("not a ZIP file (no end of central directory record)");
|
|
33
|
+
|
|
34
|
+
// Where the central directory must end: right before the (ZIP64) end record.
|
|
35
|
+
let cdEnd = eocd;
|
|
36
|
+
let entries = view.getUint16(eocd + 10, true);
|
|
37
|
+
let cdSize = view.getUint32(eocd + 12, true);
|
|
38
|
+
let cdOffset = view.getUint32(eocd + 16, true);
|
|
39
|
+
|
|
40
|
+
if (entries === 0xffff || cdSize === 0xffffffff || cdOffset === 0xffffffff) {
|
|
41
|
+
const locator = eocd - 20;
|
|
42
|
+
if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIG) {
|
|
43
|
+
const z64 = Number(view.getBigUint64(locator + 8, true));
|
|
44
|
+
if (z64 + 56 > len || view.getUint32(z64, true) !== ZIP64_EOCD_SIG) {
|
|
45
|
+
fail("bad ZIP64 end of central directory record");
|
|
46
|
+
}
|
|
47
|
+
cdEnd = z64;
|
|
48
|
+
entries = Number(view.getBigUint64(z64 + 32, true));
|
|
49
|
+
cdSize = Number(view.getBigUint64(z64 + 40, true));
|
|
50
|
+
cdOffset = Number(view.getBigUint64(z64 + 48, true));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// An archive with bytes prepended (a self-extracting stub, for example) keeps offsets relative
|
|
55
|
+
// to the original start; JSZip accepts it, so find the directory from its end instead.
|
|
56
|
+
if (cdOffset + 4 > len || (entries > 0 && view.getUint32(cdOffset, true) !== CD_ENTRY_SIG)) {
|
|
57
|
+
const shifted = cdEnd - cdSize;
|
|
58
|
+
if (shifted >= 0 && shifted + 4 <= len && view.getUint32(shifted, true) === CD_ENTRY_SIG) {
|
|
59
|
+
cdOffset = shifted;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (cdOffset + cdSize > len) fail("central directory lies outside the file");
|
|
63
|
+
|
|
64
|
+
let total = 0;
|
|
65
|
+
let pos = cdOffset;
|
|
66
|
+
for (let n = 0; n < entries; n++) {
|
|
67
|
+
if (pos + 46 > len || view.getUint32(pos, true) !== CD_ENTRY_SIG) {
|
|
68
|
+
fail("truncated or corrupt central directory");
|
|
69
|
+
}
|
|
70
|
+
const size32 = view.getUint32(pos + 24, true);
|
|
71
|
+
const nameLen = view.getUint16(pos + 28, true);
|
|
72
|
+
const extraLen = view.getUint16(pos + 30, true);
|
|
73
|
+
const commentLen = view.getUint16(pos + 32, true);
|
|
74
|
+
let size = size32;
|
|
75
|
+
if (size32 === 0xffffffff) {
|
|
76
|
+
size = readZip64Size(view, pos + 46 + nameLen, extraLen);
|
|
77
|
+
}
|
|
78
|
+
total += size;
|
|
79
|
+
pos += 46 + nameLen + extraLen + commentLen;
|
|
80
|
+
}
|
|
81
|
+
return total;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Reads the 64-bit uncompressed size from an entry's ZIP64 extended information extra field. */
|
|
85
|
+
function readZip64Size(view: DataView, start: number, length: number): number {
|
|
86
|
+
const end = start + length;
|
|
87
|
+
if (end > view.byteLength) fail("truncated or corrupt central directory");
|
|
88
|
+
let p = start;
|
|
89
|
+
while (p + 4 <= end) {
|
|
90
|
+
const id = view.getUint16(p, true);
|
|
91
|
+
const size = view.getUint16(p + 2, true);
|
|
92
|
+
if (id === 0x0001) {
|
|
93
|
+
// The uncompressed size is the first field when the 32-bit value is 0xFFFFFFFF.
|
|
94
|
+
if (size < 8 || p + 12 > end) fail("bad ZIP64 extra field");
|
|
95
|
+
return Number(view.getBigUint64(p + 4, true));
|
|
96
|
+
}
|
|
97
|
+
p += 4 + size;
|
|
98
|
+
}
|
|
99
|
+
fail("missing ZIP64 extra field");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Throws when the uncompressed size declared by a ZIP archive exceeds `maxBytes`, so a small
|
|
104
|
+
* "zip bomb" is refused before a parser inflates it.
|
|
105
|
+
*/
|
|
106
|
+
export function assertZipUncompressedSize(bytes: Uint8Array, maxBytes: number): void {
|
|
107
|
+
const size = zipUncompressedSize(bytes);
|
|
108
|
+
if (size > maxBytes) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`Archive uncompressed size ${size} bytes exceeds the limit of ${maxBytes} bytes`,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|