@statewalker/content-extractors 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -10
- package/dist/collect-bytes.d.ts +10 -2
- package/dist/collect-bytes.d.ts.map +1 -1
- package/dist/collect-bytes.js +6 -4
- package/dist/collect-bytes.js.map +1 -1
- package/dist/extractor-registry.d.ts +2 -1
- package/dist/extractor-registry.d.ts.map +1 -1
- package/dist/extractor-registry.js +3 -2
- package/dist/extractor-registry.js.map +1 -1
- package/dist/extractors/docx-extractor.d.ts +5 -1
- package/dist/extractors/docx-extractor.d.ts.map +1 -1
- package/dist/extractors/docx-extractor.js +20 -11
- package/dist/extractors/docx-extractor.js.map +1 -1
- package/dist/extractors/html-extractor.d.ts +3 -0
- package/dist/extractors/html-extractor.d.ts.map +1 -1
- package/dist/extractors/html-extractor.js +8 -5
- package/dist/extractors/html-extractor.js.map +1 -1
- package/dist/extractors/index.d.ts +12 -9
- package/dist/extractors/index.d.ts.map +1 -1
- package/dist/extractors/index.js +23 -18
- package/dist/extractors/index.js.map +1 -1
- package/dist/extractors/markdown-extractor.d.ts +3 -0
- package/dist/extractors/markdown-extractor.d.ts.map +1 -1
- package/dist/extractors/markdown-extractor.js +8 -4
- package/dist/extractors/markdown-extractor.js.map +1 -1
- package/dist/extractors/pdf-extractor.d.ts +3 -0
- package/dist/extractors/pdf-extractor.d.ts.map +1 -1
- package/dist/extractors/pdf-extractor.js +17 -11
- package/dist/extractors/pdf-extractor.js.map +1 -1
- package/dist/extractors/text-extractor.d.ts +3 -0
- package/dist/extractors/text-extractor.d.ts.map +1 -1
- package/dist/extractors/text-extractor.js +8 -4
- package/dist/extractors/text-extractor.js.map +1 -1
- package/dist/extractors/xlsx-extractor.d.ts +5 -1
- package/dist/extractors/xlsx-extractor.d.ts.map +1 -1
- package/dist/extractors/xlsx-extractor.js +56 -36
- package/dist/extractors/xlsx-extractor.js.map +1 -1
- package/dist/html-to-markdown.d.ts +3 -1
- package/dist/html-to-markdown.d.ts.map +1 -1
- package/dist/html-to-markdown.js +9 -1
- package/dist/html-to-markdown.js.map +1 -1
- package/dist/index.d.ts +11 -9
- package/dist/index.js +9 -7
- package/dist/limits.d.ts +20 -0
- package/dist/limits.d.ts.map +1 -0
- package/dist/limits.js +9 -0
- package/dist/limits.js.map +1 -0
- package/dist/markdown-to-html.d.ts +11 -3
- package/dist/markdown-to-html.d.ts.map +1 -1
- package/dist/markdown-to-html.js +13 -6
- package/dist/markdown-to-html.js.map +1 -1
- package/dist/mime-utils.js +3 -2
- package/dist/mime-utils.js.map +1 -1
- package/dist/zip-size.d.ts +16 -0
- package/dist/zip-size.d.ts.map +1 -0
- package/dist/zip-size.js +90 -0
- package/dist/zip-size.js.map +1 -0
- package/package.json +8 -4
- package/src/collect-bytes.ts +17 -2
- package/src/extractor-registry.ts +3 -2
- package/src/extractors/docx-extractor.ts +26 -11
- package/src/extractors/html-extractor.ts +8 -4
- package/src/extractors/index.ts +29 -23
- package/src/extractors/markdown-extractor.ts +8 -3
- package/src/extractors/pdf-extractor.ts +20 -11
- package/src/extractors/text-extractor.ts +8 -3
- package/src/extractors/xlsx-extractor.ts +72 -41
- package/src/html-to-markdown.ts +4 -1
- package/src/index.ts +2 -0
- package/src/limits.ts +20 -0
- package/src/markdown-to-html.ts +16 -10
- package/src/mime-utils.ts +3 -2
- package/src/zip-size.ts +113 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { collectBytes } from "../collect-bytes.js";
|
|
2
|
+
import { DEFAULT_MAX_BYTES } from "../limits.js";
|
|
2
3
|
import { extractText } from "unpdf";
|
|
3
4
|
//#region src/extractors/pdf-extractor.ts
|
|
4
5
|
/**
|
|
@@ -6,17 +7,22 @@ import { extractText } from "unpdf";
|
|
|
6
7
|
* section with a `## Page N` header. This preserves page boundaries in
|
|
7
8
|
* the indexed content, making search results more precise.
|
|
8
9
|
*/
|
|
9
|
-
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
10
|
+
function createPdfExtractor(limits = {}) {
|
|
11
|
+
const { maxBytes = DEFAULT_MAX_BYTES } = limits;
|
|
12
|
+
return async (content) => {
|
|
13
|
+
const bytes = await collectBytes(content, { maxBytes });
|
|
14
|
+
const { text: pages } = await extractText(new Uint8Array(bytes));
|
|
15
|
+
const sections = [];
|
|
16
|
+
for (let i = 0; i < pages.length; i++) {
|
|
17
|
+
const pageText = pages[i]?.trim();
|
|
18
|
+
if (pageText) sections.push(`## Page ${i + 1}\n\n${pageText}`);
|
|
19
|
+
}
|
|
20
|
+
return sections.join("\n\n");
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/** PDF extractor with the default limits. */
|
|
24
|
+
const pdfExtractor = createPdfExtractor();
|
|
19
25
|
//#endregion
|
|
20
|
-
export { pdfExtractor };
|
|
26
|
+
export { createPdfExtractor, pdfExtractor };
|
|
21
27
|
|
|
22
28
|
//# sourceMappingURL=pdf-extractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pdf-extractor.js","names":[],"sources":["../../src/extractors/pdf-extractor.ts"],"sourcesContent":["import { extractText } from \"unpdf\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Extracts PDF content page-by-page, formatting each page as a markdown\n * section with a `## Page N` header. This preserves page boundaries in\n * the indexed content, making search results more precise.\n */\nexport
|
|
1
|
+
{"version":3,"file":"pdf-extractor.js","names":[],"sources":["../../src/extractors/pdf-extractor.ts"],"sourcesContent":["import { extractText } from \"unpdf\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport { DEFAULT_MAX_BYTES, type ExtractorLimits } from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Extracts PDF content page-by-page, formatting each page as a markdown\n * section with a `## Page N` header. This preserves page boundaries in\n * the indexed content, making search results more precise.\n */\nexport function createPdfExtractor(limits: ExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES } = limits;\n return async (content) => {\n const bytes = await collectBytes(content, { maxBytes });\n // A copy, always a plain Uint8Array: pdfjs rejects a Node Buffer, and it takes over (detaches)\n // the buffer it is given, which would empty the caller's array.\n const { text: pages } = await extractText(new Uint8Array(bytes));\n\n const sections: string[] = [];\n for (let i = 0; i < pages.length; i++) {\n const pageText = pages[i]?.trim();\n if (pageText) {\n sections.push(`## Page ${i + 1}\\n\\n${pageText}`);\n }\n }\n\n return sections.join(\"\\n\\n\");\n };\n}\n\n/** PDF extractor with the default limits. */\nexport const pdfExtractor: ContentExtractor = createPdfExtractor();\n"],"mappings":";;;;;;;;;AAUA,SAAgB,mBAAmB,SAA0B,CAAC,GAAqB;CACjF,MAAM,EAAE,WAAW,sBAAsB;CACzC,OAAO,OAAO,YAAY;EACxB,MAAM,QAAQ,MAAM,aAAa,SAAS,EAAE,SAAS,CAAC;EAGtD,MAAM,EAAE,MAAM,UAAU,MAAM,YAAY,IAAI,WAAW,KAAK,CAAC;EAE/D,MAAM,WAAqB,CAAC;EAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,WAAW,MAAM,EAAE,EAAE,KAAK;GAChC,IAAI,UACF,SAAS,KAAK,WAAW,IAAI,EAAE,MAAM,UAAU;EAEnD;EAEA,OAAO,SAAS,KAAK,MAAM;CAC7B;AACF;;AAGA,MAAa,eAAiC,mBAAmB"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ExtractorLimits } from "../limits.js";
|
|
1
2
|
import { ContentExtractor } from "../types.js";
|
|
2
3
|
//#region src/extractors/text-extractor.d.ts
|
|
3
4
|
/**
|
|
@@ -5,6 +6,8 @@ import { ContentExtractor } from "../types.js";
|
|
|
5
6
|
* decode bytes to a string. Exists as a distinct entry so `.txt` files
|
|
6
7
|
* route through the same pipeline as every other format.
|
|
7
8
|
*/
|
|
9
|
+
export declare function createTextExtractor(limits?: ExtractorLimits): ContentExtractor;
|
|
10
|
+
/** Plain-text extractor with the default limits. */
|
|
8
11
|
export declare const textExtractor: ContentExtractor;
|
|
9
12
|
//#endregion
|
|
10
13
|
//# sourceMappingURL=text-extractor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text-extractor.d.ts","names":[],"sources":["../../src/extractors/text-extractor.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"text-extractor.d.ts","names":[],"sources":["../../src/extractors/text-extractor.ts"],"mappings":";;;;;;;;wBASgB,oBAAoB,SAAQ,kBAAuB;;qBAMtD,eAAe"}
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
import { collectText } from "../collect-bytes.js";
|
|
2
|
+
import { DEFAULT_MAX_BYTES } from "../limits.js";
|
|
2
3
|
//#region src/extractors/text-extractor.ts
|
|
3
4
|
/**
|
|
4
5
|
* Identity extractor for plain text -- no conversion needed, just
|
|
5
6
|
* decode bytes to a string. Exists as a distinct entry so `.txt` files
|
|
6
7
|
* route through the same pipeline as every other format.
|
|
7
8
|
*/
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
};
|
|
9
|
+
function createTextExtractor(limits = {}) {
|
|
10
|
+
const { maxBytes = DEFAULT_MAX_BYTES } = limits;
|
|
11
|
+
return (content) => collectText(content, { maxBytes });
|
|
12
|
+
}
|
|
13
|
+
/** Plain-text extractor with the default limits. */
|
|
14
|
+
const textExtractor = createTextExtractor();
|
|
11
15
|
//#endregion
|
|
12
|
-
export { textExtractor };
|
|
16
|
+
export { createTextExtractor, textExtractor };
|
|
13
17
|
|
|
14
18
|
//# sourceMappingURL=text-extractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"text-extractor.js","names":[],"sources":["../../src/extractors/text-extractor.ts"],"sourcesContent":["import { collectText } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Identity extractor for plain text -- no conversion needed, just\n * decode bytes to a string. Exists as a distinct entry so `.txt` files\n * route through the same pipeline as every other format.\n */\nexport
|
|
1
|
+
{"version":3,"file":"text-extractor.js","names":[],"sources":["../../src/extractors/text-extractor.ts"],"sourcesContent":["import { collectText } from \"../collect-bytes.js\";\nimport { DEFAULT_MAX_BYTES, type ExtractorLimits } from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Identity extractor for plain text -- no conversion needed, just\n * decode bytes to a string. Exists as a distinct entry so `.txt` files\n * route through the same pipeline as every other format.\n */\nexport function createTextExtractor(limits: ExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES } = limits;\n return (content) => collectText(content, { maxBytes });\n}\n\n/** Plain-text extractor with the default limits. */\nexport const textExtractor: ContentExtractor = createTextExtractor();\n"],"mappings":";;;;;;;;AASA,SAAgB,oBAAoB,SAA0B,CAAC,GAAqB;CAClF,MAAM,EAAE,WAAW,sBAAsB;CACzC,QAAQ,YAAY,YAAY,SAAS,EAAE,SAAS,CAAC;AACvD;;AAGA,MAAa,gBAAkC,oBAAoB"}
|
|
@@ -1,11 +1,15 @@
|
|
|
1
|
+
import { ZipExtractorLimits } from "../limits.js";
|
|
1
2
|
import { ContentExtractor } from "../types.js";
|
|
2
3
|
//#region src/extractors/xlsx-extractor.d.ts
|
|
3
4
|
/**
|
|
4
5
|
* Converts Excel spreadsheets into markdown tables so the content
|
|
5
6
|
* is searchable and consumable by text-based pipelines. Each worksheet
|
|
6
7
|
* becomes a separate section with a level-2 heading. Empty sheets are
|
|
7
|
-
* skipped to avoid noise.
|
|
8
|
+
* skipped to avoid noise. The input size and the archive's declared
|
|
9
|
+
* uncompressed size are checked before exceljs inflates it.
|
|
8
10
|
*/
|
|
11
|
+
export declare function createXlsxExtractor(limits?: ZipExtractorLimits): ContentExtractor;
|
|
12
|
+
/** XLSX extractor with the default limits. */
|
|
9
13
|
export declare const xlsxExtractor: ContentExtractor;
|
|
10
14
|
//#endregion
|
|
11
15
|
//# sourceMappingURL=xlsx-extractor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xlsx-extractor.d.ts","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"xlsx-extractor.d.ts","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"mappings":";;;;;;;;;;wBAkCgB,oBAAoB,SAAQ,qBAA0B;;qBAqDzD,eAAe"}
|
|
@@ -1,49 +1,69 @@
|
|
|
1
1
|
import { collectBytes } from "../collect-bytes.js";
|
|
2
|
+
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES } from "../limits.js";
|
|
3
|
+
import { assertZipUncompressedSize } from "../zip-size.js";
|
|
2
4
|
import ExcelJS from "exceljs";
|
|
3
5
|
//#region src/extractors/xlsx-extractor.ts
|
|
4
6
|
/**
|
|
7
|
+
* Makes a cell value safe inside a Markdown table row: `|` becomes `\|` so it does not start a
|
|
8
|
+
* new column, and every line break (CR, LF, CRLF) becomes one space so it does not end the row.
|
|
9
|
+
*/
|
|
10
|
+
function escapeCell(value) {
|
|
11
|
+
return value.replace(/\|/g, "\\|").replace(/\r\n|\r|\n/g, " ");
|
|
12
|
+
}
|
|
13
|
+
/** Width of the widest row. A loop, not `Math.max(...)`, so large sheets do not overflow the stack. */
|
|
14
|
+
function maxColumnCount(rows) {
|
|
15
|
+
let max = 0;
|
|
16
|
+
for (const row of rows) if (row.length > max) max = row.length;
|
|
17
|
+
return max;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
5
20
|
* Converts Excel spreadsheets into markdown tables so the content
|
|
6
21
|
* is searchable and consumable by text-based pipelines. Each worksheet
|
|
7
22
|
* becomes a separate section with a level-2 heading. Empty sheets are
|
|
8
|
-
* skipped to avoid noise.
|
|
23
|
+
* skipped to avoid noise. The input size and the archive's declared
|
|
24
|
+
* uncompressed size are checked before exceljs inflates it.
|
|
9
25
|
*/
|
|
10
|
-
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
function createXlsxExtractor(limits = {}) {
|
|
27
|
+
const { maxBytes = DEFAULT_MAX_BYTES, maxUncompressedBytes = DEFAULT_MAX_UNCOMPRESSED_BYTES } = limits;
|
|
28
|
+
return async (content) => {
|
|
29
|
+
const bytes = await collectBytes(content, { maxBytes });
|
|
30
|
+
assertZipUncompressedSize(bytes, maxUncompressedBytes);
|
|
31
|
+
const workbook = new ExcelJS.Workbook();
|
|
32
|
+
const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
33
|
+
await workbook.xlsx.load(arrayBuffer);
|
|
34
|
+
const sections = [];
|
|
35
|
+
workbook.eachSheet((sheet) => {
|
|
36
|
+
const rows = [];
|
|
37
|
+
sheet.eachRow((row) => {
|
|
38
|
+
const cells = [];
|
|
39
|
+
row.eachCell({ includeEmpty: true }, (cell) => {
|
|
40
|
+
try {
|
|
41
|
+
const v = cell.value;
|
|
42
|
+
cells.push(v == null ? "" : escapeCell(String(v)));
|
|
43
|
+
} catch {
|
|
44
|
+
cells.push("");
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
rows.push(cells);
|
|
27
48
|
});
|
|
28
|
-
rows.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
49
|
+
if (rows.length === 0) return;
|
|
50
|
+
const colCount = maxColumnCount(rows);
|
|
51
|
+
for (const r of rows) while (r.length < colCount) r.push("");
|
|
52
|
+
const [headers, ...dataRows] = rows;
|
|
53
|
+
if (!headers) return;
|
|
54
|
+
const table = [
|
|
55
|
+
`| ${headers.join(" | ")} |`,
|
|
56
|
+
`| ${headers.map(() => "---").join(" | ")} |`,
|
|
57
|
+
...dataRows.map((r) => `| ${r.join(" | ")} |`)
|
|
58
|
+
].join("\n");
|
|
59
|
+
sections.push(`## ${sheet.name}\n\n${table}`);
|
|
35
60
|
});
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
].join("\n");
|
|
42
|
-
sections.push(`## ${sheet.name}\n\n${table}`);
|
|
43
|
-
});
|
|
44
|
-
return sections.join("\n\n");
|
|
45
|
-
};
|
|
61
|
+
return sections.join("\n\n");
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** XLSX extractor with the default limits. */
|
|
65
|
+
const xlsxExtractor = createXlsxExtractor();
|
|
46
66
|
//#endregion
|
|
47
|
-
export { xlsxExtractor };
|
|
67
|
+
export { createXlsxExtractor, escapeCell, maxColumnCount, xlsxExtractor };
|
|
48
68
|
|
|
49
69
|
//# sourceMappingURL=xlsx-extractor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xlsx-extractor.js","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"sourcesContent":["import ExcelJS from \"exceljs\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Converts Excel spreadsheets into markdown tables so the content\n * is searchable and consumable by text-based pipelines. Each worksheet\n * becomes a separate section with a level-2 heading. Empty sheets are\n * skipped to avoid noise.\n */\nexport
|
|
1
|
+
{"version":3,"file":"xlsx-extractor.js","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"sourcesContent":["import ExcelJS from \"exceljs\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport {\n DEFAULT_MAX_BYTES,\n DEFAULT_MAX_UNCOMPRESSED_BYTES,\n type ZipExtractorLimits,\n} from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\nimport { assertZipUncompressedSize } from \"../zip-size.js\";\n\n/**\n * Makes a cell value safe inside a Markdown table row: `|` becomes `\\|` so it does not start a\n * new column, and every line break (CR, LF, CRLF) becomes one space so it does not end the row.\n */\nexport function escapeCell(value: string): string {\n return value.replace(/\\|/g, \"\\\\|\").replace(/\\r\\n|\\r|\\n/g, \" \");\n}\n\n/** Width of the widest row. A loop, not `Math.max(...)`, so large sheets do not overflow the stack. */\nexport function maxColumnCount(rows: readonly (readonly unknown[])[]): number {\n let max = 0;\n for (const row of rows) {\n if (row.length > max) max = row.length;\n }\n return max;\n}\n\n/**\n * Converts Excel spreadsheets into markdown tables so the content\n * is searchable and consumable by text-based pipelines. Each worksheet\n * becomes a separate section with a level-2 heading. Empty sheets are\n * skipped to avoid noise. The input size and the archive's declared\n * uncompressed size are checked before exceljs inflates it.\n */\nexport function createXlsxExtractor(limits: ZipExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES, maxUncompressedBytes = DEFAULT_MAX_UNCOMPRESSED_BYTES } =\n limits;\n return async (content) => {\n const bytes = await collectBytes(content, { maxBytes });\n assertZipUncompressedSize(bytes, maxUncompressedBytes);\n const workbook = new ExcelJS.Workbook();\n const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);\n await workbook.xlsx.load(arrayBuffer as ArrayBuffer);\n\n const sections: string[] = [];\n\n workbook.eachSheet((sheet) => {\n const rows: string[][] = [];\n\n sheet.eachRow((row) => {\n const cells: string[] = [];\n row.eachCell({ includeEmpty: true }, (cell) => {\n try {\n const v = cell.value;\n cells.push(v == null ? \"\" : escapeCell(String(v)));\n } catch {\n cells.push(\"\");\n }\n });\n rows.push(cells);\n });\n\n if (rows.length === 0) return;\n\n // Normalize column count to the widest row\n const colCount = maxColumnCount(rows);\n for (const r of rows) {\n while (r.length < colCount) r.push(\"\");\n }\n\n // First row as headers, rest as data\n const [headers, ...dataRows] = rows;\n if (!headers) return;\n\n const headerLine = `| ${headers.join(\" | \")} |`;\n const separator = `| ${headers.map(() => \"---\").join(\" | \")} |`;\n const bodyLines = dataRows.map((r) => `| ${r.join(\" | \")} |`);\n\n const table = [headerLine, separator, ...bodyLines].join(\"\\n\");\n sections.push(`## ${sheet.name}\\n\\n${table}`);\n });\n\n return sections.join(\"\\n\\n\");\n };\n}\n\n/** XLSX extractor with the default limits. */\nexport const xlsxExtractor: ContentExtractor = createXlsxExtractor();\n"],"mappings":";;;;;;;;;AAcA,SAAgB,WAAW,OAAuB;CAChD,OAAO,MAAM,QAAQ,OAAO,KAAK,CAAC,CAAC,QAAQ,eAAe,GAAG;AAC/D;;AAGA,SAAgB,eAAe,MAA+C;CAC5E,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,SAAS,KAAK,MAAM,IAAI;CAElC,OAAO;AACT;;;;;;;;AASA,SAAgB,oBAAoB,SAA6B,CAAC,GAAqB;CACrF,MAAM,EAAE,WAAW,mBAAmB,uBAAuB,mCAC3D;CACF,OAAO,OAAO,YAAY;EACxB,MAAM,QAAQ,MAAM,aAAa,SAAS,EAAE,SAAS,CAAC;EACtD,0BAA0B,OAAO,oBAAoB;EACrD,MAAM,WAAW,IAAI,QAAQ,SAAS;EACtC,MAAM,cAAc,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;EAC5F,MAAM,SAAS,KAAK,KAAK,WAA0B;EAEnD,MAAM,WAAqB,CAAC;EAE5B,SAAS,WAAW,UAAU;GAC5B,MAAM,OAAmB,CAAC;GAE1B,MAAM,SAAS,QAAQ;IACrB,MAAM,QAAkB,CAAC;IACzB,IAAI,SAAS,EAAE,cAAc,KAAK,IAAI,SAAS;KAC7C,IAAI;MACF,MAAM,IAAI,KAAK;MACf,MAAM,KAAK,KAAK,OAAO,KAAK,WAAW,OAAO,CAAC,CAAC,CAAC;KACnD,QAAQ;MACN,MAAM,KAAK,EAAE;KACf;IACF,CAAC;IACD,KAAK,KAAK,KAAK;GACjB,CAAC;GAED,IAAI,KAAK,WAAW,GAAG;GAGvB,MAAM,WAAW,eAAe,IAAI;GACpC,KAAK,MAAM,KAAK,MACd,OAAO,EAAE,SAAS,UAAU,EAAE,KAAK,EAAE;GAIvC,MAAM,CAAC,SAAS,GAAG,YAAY;GAC/B,IAAI,CAAC,SAAS;GAMd,MAAM,QAAQ;IAAC,KAJS,QAAQ,KAAK,KAAK,EAAE;IAIjB,KAHJ,QAAQ,UAAU,KAAK,CAAC,CAAC,KAAK,KAAK,EAAE;IAGtB,GAFpB,SAAS,KAAK,MAAM,KAAK,EAAE,KAAK,KAAK,EAAE,GAER;GAAC,CAAC,CAAC,KAAK,IAAI;GAC7D,SAAS,KAAK,MAAM,MAAM,KAAK,MAAM,OAAO;EAC9C,CAAC;EAED,OAAO,SAAS,KAAK,MAAM;CAC7B;AACF;;AAGA,MAAa,gBAAkC,oBAAoB"}
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
* Normalizes HTML into markdown so downstream consumers get a uniform
|
|
4
4
|
* text format regardless of whether the source was HTML, DOCX (via
|
|
5
5
|
* mammoth's HTML output), or any other HTML-producing extractor.
|
|
6
|
-
* Uses Turndown with GFM tables and strikethrough support.
|
|
6
|
+
* Uses Turndown with GFM tables and strikethrough support. The content of
|
|
7
|
+
* `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is
|
|
8
|
+
* code or markup, not document text.
|
|
7
9
|
*/
|
|
8
10
|
export declare function htmlToMarkdown(html: string): string;
|
|
9
11
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-to-markdown.d.ts","names":[],"sources":["../src/html-to-markdown.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"html-to-markdown.d.ts","names":[],"sources":["../src/html-to-markdown.ts"],"mappings":";;;;;;;;;wBAYgB,eAAe"}
|
package/dist/html-to-markdown.js
CHANGED
|
@@ -5,7 +5,9 @@ import TurndownService from "turndown";
|
|
|
5
5
|
* Normalizes HTML into markdown so downstream consumers get a uniform
|
|
6
6
|
* text format regardless of whether the source was HTML, DOCX (via
|
|
7
7
|
* mammoth's HTML output), or any other HTML-producing extractor.
|
|
8
|
-
* Uses Turndown with GFM tables and strikethrough support.
|
|
8
|
+
* Uses Turndown with GFM tables and strikethrough support. The content of
|
|
9
|
+
* `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is
|
|
10
|
+
* code or markup, not document text.
|
|
9
11
|
*/
|
|
10
12
|
function htmlToMarkdown(html) {
|
|
11
13
|
const turndownService = new TurndownService({
|
|
@@ -17,6 +19,12 @@ function htmlToMarkdown(html) {
|
|
|
17
19
|
strongDelimiter: "**"
|
|
18
20
|
});
|
|
19
21
|
turndownService.use(gfm);
|
|
22
|
+
turndownService.remove([
|
|
23
|
+
"script",
|
|
24
|
+
"style",
|
|
25
|
+
"noscript",
|
|
26
|
+
"template"
|
|
27
|
+
]);
|
|
20
28
|
return turndownService.turndown(html);
|
|
21
29
|
}
|
|
22
30
|
//#endregion
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"html-to-markdown.js","names":[],"sources":["../src/html-to-markdown.ts"],"sourcesContent":["/// <reference path=\"./turndown-plugin-gfm.d.ts\" />\nimport { gfm } from \"@joplin/turndown-plugin-gfm\";\nimport TurndownService from \"turndown\";\n\n/**\n * Normalizes HTML into markdown so downstream consumers get a uniform\n * text format regardless of whether the source was HTML, DOCX (via\n * mammoth's HTML output), or any other HTML-producing extractor.\n * Uses Turndown with GFM tables and strikethrough support.\n */\nexport function htmlToMarkdown(html: string): string {\n const turndownService = new TurndownService({\n headingStyle: \"atx\",\n hr: \"---\",\n bulletListMarker: \"*\",\n codeBlockStyle: \"fenced\",\n emDelimiter: \"*\",\n strongDelimiter: \"**\",\n });\n turndownService.use(gfm);\n return turndownService.turndown(html);\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"html-to-markdown.js","names":[],"sources":["../src/html-to-markdown.ts"],"sourcesContent":["/// <reference path=\"./turndown-plugin-gfm.d.ts\" />\nimport { gfm } from \"@joplin/turndown-plugin-gfm\";\nimport TurndownService from \"turndown\";\n\n/**\n * Normalizes HTML into markdown so downstream consumers get a uniform\n * text format regardless of whether the source was HTML, DOCX (via\n * mammoth's HTML output), or any other HTML-producing extractor.\n * Uses Turndown with GFM tables and strikethrough support. The content of\n * `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is\n * code or markup, not document text.\n */\nexport function htmlToMarkdown(html: string): string {\n const turndownService = new TurndownService({\n headingStyle: \"atx\",\n hr: \"---\",\n bulletListMarker: \"*\",\n codeBlockStyle: \"fenced\",\n emDelimiter: \"*\",\n strongDelimiter: \"**\",\n });\n turndownService.use(gfm);\n turndownService.remove([\"script\", \"style\", \"noscript\", \"template\"]);\n return turndownService.turndown(html);\n}\n"],"mappings":";;;;;;;;;;;AAYA,SAAgB,eAAe,MAAsB;CACnD,MAAM,kBAAkB,IAAI,gBAAgB;EAC1C,cAAc;EACd,IAAI;EACJ,kBAAkB;EAClB,gBAAgB;EAChB,aAAa;EACb,iBAAiB;CACnB,CAAC;CACD,gBAAgB,IAAI,GAAG;CACvB,gBAAgB,OAAO;EAAC;EAAU;EAAS;EAAY;CAAU,CAAC;CAClE,OAAO,gBAAgB,SAAS,IAAI;AACtC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorLimits, ZipExtractorLimits } from "./limits.js";
|
|
1
2
|
import { ContentExtractor, ContentNormalizer } from "./types.js";
|
|
2
|
-
import { docxExtractor } from "./extractors/docx-extractor.js";
|
|
3
|
-
import { htmlExtractor } from "./extractors/html-extractor.js";
|
|
4
|
-
import { markdownExtractor } from "./extractors/markdown-extractor.js";
|
|
5
|
-
import { pdfExtractor } from "./extractors/pdf-extractor.js";
|
|
6
|
-
import { textExtractor } from "./extractors/text-extractor.js";
|
|
7
|
-
import { xlsxExtractor } from "./extractors/xlsx-extractor.js";
|
|
3
|
+
import { createDocxExtractor, docxExtractor } from "./extractors/docx-extractor.js";
|
|
4
|
+
import { createHtmlExtractor, htmlExtractor } from "./extractors/html-extractor.js";
|
|
5
|
+
import { createMarkdownExtractor, markdownExtractor } from "./extractors/markdown-extractor.js";
|
|
6
|
+
import { createPdfExtractor, pdfExtractor } from "./extractors/pdf-extractor.js";
|
|
7
|
+
import { createTextExtractor, textExtractor } from "./extractors/text-extractor.js";
|
|
8
|
+
import { createXlsxExtractor, xlsxExtractor } from "./extractors/xlsx-extractor.js";
|
|
8
9
|
import { ExtractorRegistry } from "./extractor-registry.js";
|
|
9
10
|
import { createDefaultRegistry } from "./extractors/index.js";
|
|
10
|
-
import { collectBytes, collectText } from "./collect-bytes.js";
|
|
11
|
+
import { CollectOptions, collectBytes, collectText } from "./collect-bytes.js";
|
|
11
12
|
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
12
|
-
import { markdownToHtml } from "./markdown-to-html.js";
|
|
13
|
+
import { MarkdownToHtmlOptions, markdownToHtml } from "./markdown-to-html.js";
|
|
13
14
|
import { getMimeType } from "./mime-utils.js";
|
|
14
|
-
|
|
15
|
+
import { assertZipUncompressedSize, zipUncompressedSize } from "./zip-size.js";
|
|
16
|
+
export { CollectOptions, ContentExtractor, ContentNormalizer, DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorLimits, ExtractorRegistry, MarkdownToHtmlOptions, ZipExtractorLimits, assertZipUncompressedSize, collectBytes, collectText, createDefaultRegistry, createDocxExtractor, createHtmlExtractor, createMarkdownExtractor, createPdfExtractor, createTextExtractor, createXlsxExtractor, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor, zipUncompressedSize };
|
package/dist/index.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { collectBytes, collectText } from "./collect-bytes.js";
|
|
2
2
|
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
3
|
+
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES } from "./limits.js";
|
|
4
|
+
import { assertZipUncompressedSize, zipUncompressedSize } from "./zip-size.js";
|
|
5
|
+
import { createDocxExtractor, docxExtractor } from "./extractors/docx-extractor.js";
|
|
6
|
+
import { createHtmlExtractor, htmlExtractor } from "./extractors/html-extractor.js";
|
|
7
|
+
import { createMarkdownExtractor, markdownExtractor } from "./extractors/markdown-extractor.js";
|
|
8
|
+
import { createPdfExtractor, pdfExtractor } from "./extractors/pdf-extractor.js";
|
|
9
|
+
import { createTextExtractor, textExtractor } from "./extractors/text-extractor.js";
|
|
10
|
+
import { createXlsxExtractor, xlsxExtractor } from "./extractors/xlsx-extractor.js";
|
|
9
11
|
import { ExtractorRegistry } from "./extractor-registry.js";
|
|
10
12
|
import { createDefaultRegistry } from "./extractors/index.js";
|
|
11
13
|
import { markdownToHtml } from "./markdown-to-html.js";
|
|
12
14
|
import { getMimeType } from "./mime-utils.js";
|
|
13
|
-
export { ExtractorRegistry, collectBytes, collectText, createDefaultRegistry, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor };
|
|
15
|
+
export { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorRegistry, assertZipUncompressedSize, collectBytes, collectText, createDefaultRegistry, createDocxExtractor, createHtmlExtractor, createMarkdownExtractor, createPdfExtractor, createTextExtractor, createXlsxExtractor, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor, zipUncompressedSize };
|
package/dist/limits.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
//#region src/limits.d.ts
|
|
2
|
+
/** Default largest input an extractor reads: 100 MiB. */
|
|
3
|
+
export declare const DEFAULT_MAX_BYTES: number;
|
|
4
|
+
/** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */
|
|
5
|
+
export declare const DEFAULT_MAX_UNCOMPRESSED_BYTES: number;
|
|
6
|
+
/** Limits every built-in extractor accepts. Pass `Infinity` to turn a limit off. */
|
|
7
|
+
export interface ExtractorLimits {
|
|
8
|
+
/** Largest input read, in bytes. Default {@link DEFAULT_MAX_BYTES}. */
|
|
9
|
+
maxBytes?: number;
|
|
10
|
+
}
|
|
11
|
+
/** Limits of the ZIP-based extractors (DOCX, DOCM, XLSX). */
|
|
12
|
+
export interface ZipExtractorLimits extends ExtractorLimits {
|
|
13
|
+
/**
|
|
14
|
+
* Largest sum of the uncompressed entry sizes declared in the archive's central directory,
|
|
15
|
+
* checked before the archive is inflated. Default {@link DEFAULT_MAX_UNCOMPRESSED_BYTES}.
|
|
16
|
+
*/
|
|
17
|
+
maxUncompressedBytes?: number;
|
|
18
|
+
}
|
|
19
|
+
//#endregion
|
|
20
|
+
//# sourceMappingURL=limits.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limits.d.ts","names":[],"sources":["../src/limits.ts"],"mappings":";;qBACa;;qBAGA;;iBAGI;;EAEf;;;iBAIe,2BAA2B;;;;;EAK1C"}
|
package/dist/limits.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
//#region src/limits.ts
|
|
2
|
+
/** Default largest input an extractor reads: 100 MiB. */
|
|
3
|
+
const DEFAULT_MAX_BYTES = 104857600;
|
|
4
|
+
/** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */
|
|
5
|
+
const DEFAULT_MAX_UNCOMPRESSED_BYTES = 524288e3;
|
|
6
|
+
//#endregion
|
|
7
|
+
export { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES };
|
|
8
|
+
|
|
9
|
+
//# sourceMappingURL=limits.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limits.js","names":[],"sources":["../src/limits.ts"],"sourcesContent":["/** Default largest input an extractor reads: 100 MiB. */\nexport const DEFAULT_MAX_BYTES = 100 * 1024 * 1024;\n\n/** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */\nexport const DEFAULT_MAX_UNCOMPRESSED_BYTES = 500 * 1024 * 1024;\n\n/** Limits every built-in extractor accepts. Pass `Infinity` to turn a limit off. */\nexport interface ExtractorLimits {\n /** Largest input read, in bytes. Default {@link DEFAULT_MAX_BYTES}. */\n maxBytes?: number;\n}\n\n/** Limits of the ZIP-based extractors (DOCX, DOCM, XLSX). */\nexport interface ZipExtractorLimits extends ExtractorLimits {\n /**\n * Largest sum of the uncompressed entry sizes declared in the archive's central directory,\n * checked before the archive is inflated. Default {@link DEFAULT_MAX_UNCOMPRESSED_BYTES}.\n */\n maxUncompressedBytes?: number;\n}\n"],"mappings":";;AACA,MAAa,oBAAoB;;AAGjC,MAAa,iCAAiC"}
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
//#region src/markdown-to-html.d.ts
|
|
2
|
+
export interface MarkdownToHtmlOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Pass raw HTML in the Markdown through to the output. Off by default: raw HTML is escaped,
|
|
5
|
+
* so `<script>` or `<img onerror>` in an uploaded file cannot run when the output is shown.
|
|
6
|
+
* Turn it on only for trusted input, or sanitise the output before rendering it.
|
|
7
|
+
*/
|
|
8
|
+
html?: boolean;
|
|
9
|
+
}
|
|
2
10
|
/**
|
|
3
11
|
* Provides the reverse of `htmlToMarkdown` for cases where extracted
|
|
4
12
|
* markdown needs to be rendered back to HTML (e.g. previews, email
|
|
5
|
-
* output).
|
|
6
|
-
*
|
|
13
|
+
* output). Bare URLs are auto-linked. Raw HTML is escaped unless
|
|
14
|
+
* `options.html` is set.
|
|
7
15
|
*/
|
|
8
|
-
export declare function markdownToHtml(markdown: string): string;
|
|
16
|
+
export declare function markdownToHtml(markdown: string, options?: MarkdownToHtmlOptions): string;
|
|
9
17
|
//#endregion
|
|
10
18
|
//# sourceMappingURL=markdown-to-html.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-to-html.d.ts","names":[],"sources":["../src/markdown-to-html.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"markdown-to-html.d.ts","names":[],"sources":["../src/markdown-to-html.ts"],"mappings":";iBAMiB;;;;;;EAMf;;;;;;;;wBASc,eAAe,kBAAkB,UAAS"}
|
package/dist/markdown-to-html.js
CHANGED
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
import MarkdownIt from "markdown-it";
|
|
2
2
|
//#region src/markdown-to-html.ts
|
|
3
|
-
const
|
|
4
|
-
html: true,
|
|
3
|
+
const options = {
|
|
5
4
|
linkify: true,
|
|
6
5
|
typographer: true,
|
|
7
6
|
breaks: false
|
|
7
|
+
};
|
|
8
|
+
const escaping = new MarkdownIt({
|
|
9
|
+
...options,
|
|
10
|
+
html: false
|
|
11
|
+
});
|
|
12
|
+
const passThrough = new MarkdownIt({
|
|
13
|
+
...options,
|
|
14
|
+
html: true
|
|
8
15
|
});
|
|
9
16
|
/**
|
|
10
17
|
* Provides the reverse of `htmlToMarkdown` for cases where extracted
|
|
11
18
|
* markdown needs to be rendered back to HTML (e.g. previews, email
|
|
12
|
-
* output).
|
|
13
|
-
*
|
|
19
|
+
* output). Bare URLs are auto-linked. Raw HTML is escaped unless
|
|
20
|
+
* `options.html` is set.
|
|
14
21
|
*/
|
|
15
|
-
function markdownToHtml(markdown) {
|
|
22
|
+
function markdownToHtml(markdown, options = {}) {
|
|
16
23
|
if (!markdown || markdown.trim() === "") return "";
|
|
17
|
-
return
|
|
24
|
+
return (options.html ? passThrough : escaping).render(markdown);
|
|
18
25
|
}
|
|
19
26
|
//#endregion
|
|
20
27
|
export { markdownToHtml };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-to-html.js","names":[],"sources":["../src/markdown-to-html.ts"],"sourcesContent":["import MarkdownIt from \"markdown-it\";\n\nconst
|
|
1
|
+
{"version":3,"file":"markdown-to-html.js","names":[],"sources":["../src/markdown-to-html.ts"],"sourcesContent":["import MarkdownIt from \"markdown-it\";\n\nconst options = { linkify: true, typographer: true, breaks: false } as const;\nconst escaping = new MarkdownIt({ ...options, html: false });\nconst passThrough = new MarkdownIt({ ...options, html: true });\n\nexport interface MarkdownToHtmlOptions {\n /**\n * Pass raw HTML in the Markdown through to the output. Off by default: raw HTML is escaped,\n * so `<script>` or `<img onerror>` in an uploaded file cannot run when the output is shown.\n * Turn it on only for trusted input, or sanitise the output before rendering it.\n */\n html?: boolean;\n}\n\n/**\n * Provides the reverse of `htmlToMarkdown` for cases where extracted\n * markdown needs to be rendered back to HTML (e.g. previews, email\n * output). Bare URLs are auto-linked. Raw HTML is escaped unless\n * `options.html` is set.\n */\nexport function markdownToHtml(markdown: string, options: MarkdownToHtmlOptions = {}): string {\n if (!markdown || markdown.trim() === \"\") {\n return \"\";\n }\n return (options.html ? passThrough : escaping).render(markdown);\n}\n"],"mappings":";;AAEA,MAAM,UAAU;CAAE,SAAS;CAAM,aAAa;CAAM,QAAQ;AAAM;AAClE,MAAM,WAAW,IAAI,WAAW;CAAE,GAAG;CAAS,MAAM;AAAM,CAAC;AAC3D,MAAM,cAAc,IAAI,WAAW;CAAE,GAAG;CAAS,MAAM;AAAK,CAAC;;;;;;;AAiB7D,SAAgB,eAAe,UAAkB,UAAiC,CAAC,GAAW;CAC5F,IAAI,CAAC,YAAY,SAAS,KAAK,MAAM,IACnC,OAAO;CAET,QAAQ,QAAQ,OAAO,cAAc,SAAA,CAAU,OAAO,QAAQ;AAChE"}
|
package/dist/mime-utils.js
CHANGED
|
@@ -18,9 +18,10 @@ const MIME_MAP = {
|
|
|
18
18
|
* Content-Type header and need a MIME type for the registry fallback path.
|
|
19
19
|
*/
|
|
20
20
|
function getMimeType(path) {
|
|
21
|
-
const
|
|
21
|
+
const name = path.slice(path.lastIndexOf("/") + 1);
|
|
22
|
+
const dotIndex = name.lastIndexOf(".");
|
|
22
23
|
if (dotIndex === -1) return void 0;
|
|
23
|
-
const ext =
|
|
24
|
+
const ext = name.slice(dotIndex).toLowerCase();
|
|
24
25
|
return MIME_MAP[ext];
|
|
25
26
|
}
|
|
26
27
|
//#endregion
|
package/dist/mime-utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mime-utils.js","names":[],"sources":["../src/mime-utils.ts"],"sourcesContent":["const MIME_MAP: Record<string, string> = {\n \".md\": \"text/markdown\",\n \".markdown\": \"text/markdown\",\n \".txt\": \"text/plain\",\n \".html\": \"text/html\",\n \".htm\": \"text/html\",\n \".json\": \"application/json\",\n \".pdf\": \"application/pdf\",\n \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n \".docm\": \"application/vnd.ms-word.document.macroEnabled.12\",\n \".xlsx\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n};\n\n/**\n * Lightweight alternative to a full MIME-sniffing library -- covers only\n * the extensions this package can actually extract, keeping the dependency\n * footprint small. Used by callers who have a filename but no\n * Content-Type header and need a MIME type for the registry fallback path.\n */\nexport function getMimeType(path: string): string | undefined {\n const
|
|
1
|
+
{"version":3,"file":"mime-utils.js","names":[],"sources":["../src/mime-utils.ts"],"sourcesContent":["const MIME_MAP: Record<string, string> = {\n \".md\": \"text/markdown\",\n \".markdown\": \"text/markdown\",\n \".txt\": \"text/plain\",\n \".html\": \"text/html\",\n \".htm\": \"text/html\",\n \".json\": \"application/json\",\n \".pdf\": \"application/pdf\",\n \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n \".docm\": \"application/vnd.ms-word.document.macroEnabled.12\",\n \".xlsx\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n};\n\n/**\n * Lightweight alternative to a full MIME-sniffing library -- covers only\n * the extensions this package can actually extract, keeping the dependency\n * footprint small. Used by callers who have a filename but no\n * Content-Type header and need a MIME type for the registry fallback path.\n */\nexport function getMimeType(path: string): string | undefined {\n const name = path.slice(path.lastIndexOf(\"/\") + 1);\n const dotIndex = name.lastIndexOf(\".\");\n if (dotIndex === -1) return undefined;\n const ext = name.slice(dotIndex).toLowerCase();\n return MIME_MAP[ext];\n}\n"],"mappings":";AAAA,MAAM,WAAmC;CACvC,OAAO;CACP,aAAa;CACb,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,SAAS;CACT,SAAS;AACX;;;;;;;AAQA,SAAgB,YAAY,MAAkC;CAC5D,MAAM,OAAO,KAAK,MAAM,KAAK,YAAY,GAAG,IAAI,CAAC;CACjD,MAAM,WAAW,KAAK,YAAY,GAAG;CACrC,IAAI,aAAa,IAAI,OAAO,KAAA;CAC5B,MAAM,MAAM,KAAK,MAAM,QAAQ,CAAC,CAAC,YAAY;CAC7C,OAAO,SAAS;AAClB"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
//#region src/zip-size.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Sums the uncompressed sizes of all entries declared in a ZIP archive's central directory,
|
|
4
|
+
* without inflating anything. Supports ZIP64. Throws when the bytes are not a readable ZIP.
|
|
5
|
+
*
|
|
6
|
+
* The sizes are what the archive claims; a parser that inflates the data still has to check
|
|
7
|
+
* them, which JSZip (used by mammoth and exceljs) does.
|
|
8
|
+
*/
|
|
9
|
+
export declare function zipUncompressedSize(bytes: Uint8Array): number;
|
|
10
|
+
/**
|
|
11
|
+
* Throws when the uncompressed size declared by a ZIP archive exceeds `maxBytes`, so a small
|
|
12
|
+
* "zip bomb" is refused before a parser inflates it.
|
|
13
|
+
*/
|
|
14
|
+
export declare function assertZipUncompressedSize(bytes: Uint8Array, maxBytes: number): void;
|
|
15
|
+
//#endregion
|
|
16
|
+
//# sourceMappingURL=zip-size.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zip-size.d.ts","names":[],"sources":["../src/zip-size.ts"],"mappings":";;;;;;;;wBAkBgB,oBAAoB,OAAO;;;;;wBAuF3B,0BAA0B,OAAO,YAAY"}
|
package/dist/zip-size.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
//#region src/zip-size.ts
|
|
2
|
+
const EOCD_SIG = 101010256;
|
|
3
|
+
const ZIP64_LOCATOR_SIG = 117853008;
|
|
4
|
+
const ZIP64_EOCD_SIG = 101075792;
|
|
5
|
+
const CD_ENTRY_SIG = 33639248;
|
|
6
|
+
const EOCD_SIZE = 22;
|
|
7
|
+
const MAX_COMMENT = 65535;
|
|
8
|
+
function fail(message) {
|
|
9
|
+
throw new Error(`Invalid ZIP archive: ${message}`);
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Sums the uncompressed sizes of all entries declared in a ZIP archive's central directory,
|
|
13
|
+
* without inflating anything. Supports ZIP64. Throws when the bytes are not a readable ZIP.
|
|
14
|
+
*
|
|
15
|
+
* The sizes are what the archive claims; a parser that inflates the data still has to check
|
|
16
|
+
* them, which JSZip (used by mammoth and exceljs) does.
|
|
17
|
+
*/
|
|
18
|
+
function zipUncompressedSize(bytes) {
|
|
19
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
20
|
+
const len = bytes.byteLength;
|
|
21
|
+
let eocd = -1;
|
|
22
|
+
const stop = Math.max(0, len - EOCD_SIZE - MAX_COMMENT);
|
|
23
|
+
for (let i = len - EOCD_SIZE; i >= stop; i--) if (view.getUint32(i, true) === EOCD_SIG) {
|
|
24
|
+
eocd = i;
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
if (eocd < 0) fail("not a ZIP file (no end of central directory record)");
|
|
28
|
+
let cdEnd = eocd;
|
|
29
|
+
let entries = view.getUint16(eocd + 10, true);
|
|
30
|
+
let cdSize = view.getUint32(eocd + 12, true);
|
|
31
|
+
let cdOffset = view.getUint32(eocd + 16, true);
|
|
32
|
+
if (entries === 65535 || cdSize === 4294967295 || cdOffset === 4294967295) {
|
|
33
|
+
const locator = eocd - 20;
|
|
34
|
+
if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIG) {
|
|
35
|
+
const z64 = Number(view.getBigUint64(locator + 8, true));
|
|
36
|
+
if (z64 + 56 > len || view.getUint32(z64, true) !== ZIP64_EOCD_SIG) fail("bad ZIP64 end of central directory record");
|
|
37
|
+
cdEnd = z64;
|
|
38
|
+
entries = Number(view.getBigUint64(z64 + 32, true));
|
|
39
|
+
cdSize = Number(view.getBigUint64(z64 + 40, true));
|
|
40
|
+
cdOffset = Number(view.getBigUint64(z64 + 48, true));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
if (cdOffset + 4 > len || entries > 0 && view.getUint32(cdOffset, true) !== CD_ENTRY_SIG) {
|
|
44
|
+
const shifted = cdEnd - cdSize;
|
|
45
|
+
if (shifted >= 0 && shifted + 4 <= len && view.getUint32(shifted, true) === CD_ENTRY_SIG) cdOffset = shifted;
|
|
46
|
+
}
|
|
47
|
+
if (cdOffset + cdSize > len) fail("central directory lies outside the file");
|
|
48
|
+
let total = 0;
|
|
49
|
+
let pos = cdOffset;
|
|
50
|
+
for (let n = 0; n < entries; n++) {
|
|
51
|
+
if (pos + 46 > len || view.getUint32(pos, true) !== CD_ENTRY_SIG) fail("truncated or corrupt central directory");
|
|
52
|
+
const size32 = view.getUint32(pos + 24, true);
|
|
53
|
+
const nameLen = view.getUint16(pos + 28, true);
|
|
54
|
+
const extraLen = view.getUint16(pos + 30, true);
|
|
55
|
+
const commentLen = view.getUint16(pos + 32, true);
|
|
56
|
+
let size = size32;
|
|
57
|
+
if (size32 === 4294967295) size = readZip64Size(view, pos + 46 + nameLen, extraLen);
|
|
58
|
+
total += size;
|
|
59
|
+
pos += 46 + nameLen + extraLen + commentLen;
|
|
60
|
+
}
|
|
61
|
+
return total;
|
|
62
|
+
}
|
|
63
|
+
/** Reads the 64-bit uncompressed size from an entry's ZIP64 extended information extra field. */
|
|
64
|
+
function readZip64Size(view, start, length) {
|
|
65
|
+
const end = start + length;
|
|
66
|
+
if (end > view.byteLength) fail("truncated or corrupt central directory");
|
|
67
|
+
let p = start;
|
|
68
|
+
while (p + 4 <= end) {
|
|
69
|
+
const id = view.getUint16(p, true);
|
|
70
|
+
const size = view.getUint16(p + 2, true);
|
|
71
|
+
if (id === 1) {
|
|
72
|
+
if (size < 8 || p + 12 > end) fail("bad ZIP64 extra field");
|
|
73
|
+
return Number(view.getBigUint64(p + 4, true));
|
|
74
|
+
}
|
|
75
|
+
p += 4 + size;
|
|
76
|
+
}
|
|
77
|
+
fail("missing ZIP64 extra field");
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Throws when the uncompressed size declared by a ZIP archive exceeds `maxBytes`, so a small
|
|
81
|
+
* "zip bomb" is refused before a parser inflates it.
|
|
82
|
+
*/
|
|
83
|
+
function assertZipUncompressedSize(bytes, maxBytes) {
|
|
84
|
+
const size = zipUncompressedSize(bytes);
|
|
85
|
+
if (size > maxBytes) throw new Error(`Archive uncompressed size ${size} bytes exceeds the limit of ${maxBytes} bytes`);
|
|
86
|
+
}
|
|
87
|
+
//#endregion
|
|
88
|
+
export { assertZipUncompressedSize, zipUncompressedSize };
|
|
89
|
+
|
|
90
|
+
//# sourceMappingURL=zip-size.js.map
|