@statewalker/content-extractors 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +105 -10
  2. package/dist/collect-bytes.d.ts +10 -2
  3. package/dist/collect-bytes.d.ts.map +1 -1
  4. package/dist/collect-bytes.js +6 -4
  5. package/dist/collect-bytes.js.map +1 -1
  6. package/dist/extractor-registry.d.ts +2 -1
  7. package/dist/extractor-registry.d.ts.map +1 -1
  8. package/dist/extractor-registry.js +3 -2
  9. package/dist/extractor-registry.js.map +1 -1
  10. package/dist/extractors/docx-extractor.d.ts +5 -1
  11. package/dist/extractors/docx-extractor.d.ts.map +1 -1
  12. package/dist/extractors/docx-extractor.js +20 -11
  13. package/dist/extractors/docx-extractor.js.map +1 -1
  14. package/dist/extractors/html-extractor.d.ts +3 -0
  15. package/dist/extractors/html-extractor.d.ts.map +1 -1
  16. package/dist/extractors/html-extractor.js +8 -5
  17. package/dist/extractors/html-extractor.js.map +1 -1
  18. package/dist/extractors/index.d.ts +12 -9
  19. package/dist/extractors/index.d.ts.map +1 -1
  20. package/dist/extractors/index.js +23 -18
  21. package/dist/extractors/index.js.map +1 -1
  22. package/dist/extractors/markdown-extractor.d.ts +3 -0
  23. package/dist/extractors/markdown-extractor.d.ts.map +1 -1
  24. package/dist/extractors/markdown-extractor.js +8 -4
  25. package/dist/extractors/markdown-extractor.js.map +1 -1
  26. package/dist/extractors/pdf-extractor.d.ts +3 -0
  27. package/dist/extractors/pdf-extractor.d.ts.map +1 -1
  28. package/dist/extractors/pdf-extractor.js +17 -11
  29. package/dist/extractors/pdf-extractor.js.map +1 -1
  30. package/dist/extractors/text-extractor.d.ts +3 -0
  31. package/dist/extractors/text-extractor.d.ts.map +1 -1
  32. package/dist/extractors/text-extractor.js +8 -4
  33. package/dist/extractors/text-extractor.js.map +1 -1
  34. package/dist/extractors/xlsx-extractor.d.ts +5 -1
  35. package/dist/extractors/xlsx-extractor.d.ts.map +1 -1
  36. package/dist/extractors/xlsx-extractor.js +56 -36
  37. package/dist/extractors/xlsx-extractor.js.map +1 -1
  38. package/dist/html-to-markdown.d.ts +3 -1
  39. package/dist/html-to-markdown.d.ts.map +1 -1
  40. package/dist/html-to-markdown.js +9 -1
  41. package/dist/html-to-markdown.js.map +1 -1
  42. package/dist/index.d.ts +11 -9
  43. package/dist/index.js +9 -7
  44. package/dist/limits.d.ts +20 -0
  45. package/dist/limits.d.ts.map +1 -0
  46. package/dist/limits.js +9 -0
  47. package/dist/limits.js.map +1 -0
  48. package/dist/markdown-to-html.d.ts +11 -3
  49. package/dist/markdown-to-html.d.ts.map +1 -1
  50. package/dist/markdown-to-html.js +13 -6
  51. package/dist/markdown-to-html.js.map +1 -1
  52. package/dist/mime-utils.js +3 -2
  53. package/dist/mime-utils.js.map +1 -1
  54. package/dist/zip-size.d.ts +16 -0
  55. package/dist/zip-size.d.ts.map +1 -0
  56. package/dist/zip-size.js +90 -0
  57. package/dist/zip-size.js.map +1 -0
  58. package/package.json +8 -4
  59. package/src/collect-bytes.ts +17 -2
  60. package/src/extractor-registry.ts +3 -2
  61. package/src/extractors/docx-extractor.ts +26 -11
  62. package/src/extractors/html-extractor.ts +8 -4
  63. package/src/extractors/index.ts +29 -23
  64. package/src/extractors/markdown-extractor.ts +8 -3
  65. package/src/extractors/pdf-extractor.ts +20 -11
  66. package/src/extractors/text-extractor.ts +8 -3
  67. package/src/extractors/xlsx-extractor.ts +72 -41
  68. package/src/html-to-markdown.ts +4 -1
  69. package/src/index.ts +2 -0
  70. package/src/limits.ts +20 -0
  71. package/src/markdown-to-html.ts +16 -10
  72. package/src/mime-utils.ts +3 -2
  73. package/src/zip-size.ts +113 -0
@@ -1,4 +1,5 @@
1
1
  import { collectBytes } from "../collect-bytes.js";
2
+ import { DEFAULT_MAX_BYTES } from "../limits.js";
2
3
  import { extractText } from "unpdf";
3
4
  //#region src/extractors/pdf-extractor.ts
4
5
  /**
@@ -6,17 +7,22 @@ import { extractText } from "unpdf";
6
7
  * section with a `## Page N` header. This preserves page boundaries in
7
8
  * the indexed content, making search results more precise.
8
9
  */
9
- const pdfExtractor = async (content) => {
10
- const bytes = await collectBytes(content);
11
- const { text: pages } = await extractText(bytes);
12
- const sections = [];
13
- for (let i = 0; i < pages.length; i++) {
14
- const pageText = pages[i]?.trim();
15
- if (pageText) sections.push(`## Page ${i + 1}\n\n${pageText}`);
16
- }
17
- return sections.join("\n\n");
18
- };
10
+ function createPdfExtractor(limits = {}) {
11
+ const { maxBytes = DEFAULT_MAX_BYTES } = limits;
12
+ return async (content) => {
13
+ const bytes = await collectBytes(content, { maxBytes });
14
+ const { text: pages } = await extractText(new Uint8Array(bytes));
15
+ const sections = [];
16
+ for (let i = 0; i < pages.length; i++) {
17
+ const pageText = pages[i]?.trim();
18
+ if (pageText) sections.push(`## Page ${i + 1}\n\n${pageText}`);
19
+ }
20
+ return sections.join("\n\n");
21
+ };
22
+ }
23
+ /** PDF extractor with the default limits. */
24
+ const pdfExtractor = createPdfExtractor();
19
25
  //#endregion
20
- export { pdfExtractor };
26
+ export { createPdfExtractor, pdfExtractor };
21
27
 
22
28
  //# sourceMappingURL=pdf-extractor.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"pdf-extractor.js","names":[],"sources":["../../src/extractors/pdf-extractor.ts"],"sourcesContent":["import { extractText } from \"unpdf\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Extracts PDF content page-by-page, formatting each page as a markdown\n * section with a `## Page N` header. This preserves page boundaries in\n * the indexed content, making search results more precise.\n */\nexport const pdfExtractor: ContentExtractor = async (content) => {\n const bytes = await collectBytes(content);\n const { text: pages } = await extractText(bytes);\n\n const sections: string[] = [];\n for (let i = 0; i < pages.length; i++) {\n const pageText = pages[i]?.trim();\n if (pageText) {\n sections.push(`## Page ${i + 1}\\n\\n${pageText}`);\n }\n }\n\n return sections.join(\"\\n\\n\");\n};\n"],"mappings":";;;;;;;;AASA,MAAa,eAAiC,OAAO,YAAY;CAC/D,MAAM,QAAQ,MAAM,aAAa,OAAO;CACxC,MAAM,EAAE,MAAM,UAAU,MAAM,YAAY,KAAK;CAE/C,MAAM,WAAqB,CAAC;CAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,WAAW,MAAM,EAAE,EAAE,KAAK;EAChC,IAAI,UACF,SAAS,KAAK,WAAW,IAAI,EAAE,MAAM,UAAU;CAEnD;CAEA,OAAO,SAAS,KAAK,MAAM;AAC7B"}
1
+ {"version":3,"file":"pdf-extractor.js","names":[],"sources":["../../src/extractors/pdf-extractor.ts"],"sourcesContent":["import { extractText } from \"unpdf\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport { DEFAULT_MAX_BYTES, type ExtractorLimits } from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Extracts PDF content page-by-page, formatting each page as a markdown\n * section with a `## Page N` header. This preserves page boundaries in\n * the indexed content, making search results more precise.\n */\nexport function createPdfExtractor(limits: ExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES } = limits;\n return async (content) => {\n const bytes = await collectBytes(content, { maxBytes });\n // A copy, always a plain Uint8Array: pdfjs rejects a Node Buffer, and it takes over (detaches)\n // the buffer it is given, which would empty the caller's array.\n const { text: pages } = await extractText(new Uint8Array(bytes));\n\n const sections: string[] = [];\n for (let i = 0; i < pages.length; i++) {\n const pageText = pages[i]?.trim();\n if (pageText) {\n sections.push(`## Page ${i + 1}\\n\\n${pageText}`);\n }\n }\n\n return sections.join(\"\\n\\n\");\n };\n}\n\n/** PDF extractor with the default limits. */\nexport const pdfExtractor: ContentExtractor = createPdfExtractor();\n"],"mappings":";;;;;;;;;AAUA,SAAgB,mBAAmB,SAA0B,CAAC,GAAqB;CACjF,MAAM,EAAE,WAAW,sBAAsB;CACzC,OAAO,OAAO,YAAY;EACxB,MAAM,QAAQ,MAAM,aAAa,SAAS,EAAE,SAAS,CAAC;EAGtD,MAAM,EAAE,MAAM,UAAU,MAAM,YAAY,IAAI,WAAW,KAAK,CAAC;EAE/D,MAAM,WAAqB,CAAC;EAC5B,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,WAAW,MAAM,EAAE,EAAE,KAAK;GAChC,IAAI,UACF,SAAS,KAAK,WAAW,IAAI,EAAE,MAAM,UAAU;EAEnD;EAEA,OAAO,SAAS,KAAK,MAAM;CAC7B;AACF;;AAGA,MAAa,eAAiC,mBAAmB"}
@@ -1,3 +1,4 @@
1
+ import { ExtractorLimits } from "../limits.js";
1
2
  import { ContentExtractor } from "../types.js";
2
3
  //#region src/extractors/text-extractor.d.ts
3
4
  /**
@@ -5,6 +6,8 @@ import { ContentExtractor } from "../types.js";
5
6
  * decode bytes to a string. Exists as a distinct entry so `.txt` files
6
7
  * route through the same pipeline as every other format.
7
8
  */
9
+ export declare function createTextExtractor(limits?: ExtractorLimits): ContentExtractor;
10
+ /** Plain-text extractor with the default limits. */
8
11
  export declare const textExtractor: ContentExtractor;
9
12
  //#endregion
10
13
  //# sourceMappingURL=text-extractor.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"text-extractor.d.ts","names":[],"sources":["../../src/extractors/text-extractor.ts"],"mappings":";;;;;;;qBAQa,eAAe"}
1
+ {"version":3,"file":"text-extractor.d.ts","names":[],"sources":["../../src/extractors/text-extractor.ts"],"mappings":";;;;;;;;wBASgB,oBAAoB,SAAQ,kBAAuB;;qBAMtD,eAAe"}
@@ -1,14 +1,18 @@
1
1
  import { collectText } from "../collect-bytes.js";
2
+ import { DEFAULT_MAX_BYTES } from "../limits.js";
2
3
  //#region src/extractors/text-extractor.ts
3
4
  /**
4
5
  * Identity extractor for plain text -- no conversion needed, just
5
6
  * decode bytes to a string. Exists as a distinct entry so `.txt` files
6
7
  * route through the same pipeline as every other format.
7
8
  */
8
- const textExtractor = async (content) => {
9
- return collectText(content);
10
- };
9
+ function createTextExtractor(limits = {}) {
10
+ const { maxBytes = DEFAULT_MAX_BYTES } = limits;
11
+ return (content) => collectText(content, { maxBytes });
12
+ }
13
+ /** Plain-text extractor with the default limits. */
14
+ const textExtractor = createTextExtractor();
11
15
  //#endregion
12
- export { textExtractor };
16
+ export { createTextExtractor, textExtractor };
13
17
 
14
18
  //# sourceMappingURL=text-extractor.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"text-extractor.js","names":[],"sources":["../../src/extractors/text-extractor.ts"],"sourcesContent":["import { collectText } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Identity extractor for plain text -- no conversion needed, just\n * decode bytes to a string. Exists as a distinct entry so `.txt` files\n * route through the same pipeline as every other format.\n */\nexport const textExtractor: ContentExtractor = async (content) => {\n return collectText(content);\n};\n"],"mappings":";;;;;;;AAQA,MAAa,gBAAkC,OAAO,YAAY;CAChE,OAAO,YAAY,OAAO;AAC5B"}
1
+ {"version":3,"file":"text-extractor.js","names":[],"sources":["../../src/extractors/text-extractor.ts"],"sourcesContent":["import { collectText } from \"../collect-bytes.js\";\nimport { DEFAULT_MAX_BYTES, type ExtractorLimits } from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Identity extractor for plain text -- no conversion needed, just\n * decode bytes to a string. Exists as a distinct entry so `.txt` files\n * route through the same pipeline as every other format.\n */\nexport function createTextExtractor(limits: ExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES } = limits;\n return (content) => collectText(content, { maxBytes });\n}\n\n/** Plain-text extractor with the default limits. */\nexport const textExtractor: ContentExtractor = createTextExtractor();\n"],"mappings":";;;;;;;;AASA,SAAgB,oBAAoB,SAA0B,CAAC,GAAqB;CAClF,MAAM,EAAE,WAAW,sBAAsB;CACzC,QAAQ,YAAY,YAAY,SAAS,EAAE,SAAS,CAAC;AACvD;;AAGA,MAAa,gBAAkC,oBAAoB"}
@@ -1,11 +1,15 @@
1
+ import { ZipExtractorLimits } from "../limits.js";
1
2
  import { ContentExtractor } from "../types.js";
2
3
  //#region src/extractors/xlsx-extractor.d.ts
3
4
  /**
4
5
  * Converts Excel spreadsheets into markdown tables so the content
5
6
  * is searchable and consumable by text-based pipelines. Each worksheet
6
7
  * becomes a separate section with a level-2 heading. Empty sheets are
7
- * skipped to avoid noise.
8
+ * skipped to avoid noise. The input size and the archive's declared
9
+ * uncompressed size are checked before exceljs inflates it.
8
10
  */
11
+ export declare function createXlsxExtractor(limits?: ZipExtractorLimits): ContentExtractor;
12
+ /** XLSX extractor with the default limits. */
9
13
  export declare const xlsxExtractor: ContentExtractor;
10
14
  //#endregion
11
15
  //# sourceMappingURL=xlsx-extractor.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"xlsx-extractor.d.ts","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"mappings":";;;;;;;;qBAUa,eAAe"}
1
+ {"version":3,"file":"xlsx-extractor.d.ts","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"mappings":";;;;;;;;;;wBAkCgB,oBAAoB,SAAQ,qBAA0B;;qBAqDzD,eAAe"}
@@ -1,49 +1,69 @@
1
1
  import { collectBytes } from "../collect-bytes.js";
2
+ import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES } from "../limits.js";
3
+ import { assertZipUncompressedSize } from "../zip-size.js";
2
4
  import ExcelJS from "exceljs";
3
5
  //#region src/extractors/xlsx-extractor.ts
4
6
  /**
7
+ * Makes a cell value safe inside a Markdown table row: `|` becomes `\|` so it does not start a
8
+ * new column, and every line break (CR, LF, CRLF) becomes one space so it does not end the row.
9
+ */
10
+ function escapeCell(value) {
11
+ return value.replace(/\|/g, "\\|").replace(/\r\n|\r|\n/g, " ");
12
+ }
13
+ /** Width of the widest row. A loop, not `Math.max(...)`, so large sheets do not overflow the stack. */
14
+ function maxColumnCount(rows) {
15
+ let max = 0;
16
+ for (const row of rows) if (row.length > max) max = row.length;
17
+ return max;
18
+ }
19
+ /**
5
20
  * Converts Excel spreadsheets into markdown tables so the content
6
21
  * is searchable and consumable by text-based pipelines. Each worksheet
7
22
  * becomes a separate section with a level-2 heading. Empty sheets are
8
- * skipped to avoid noise.
23
+ * skipped to avoid noise. The input size and the archive's declared
24
+ * uncompressed size are checked before exceljs inflates it.
9
25
  */
10
- const xlsxExtractor = async (content) => {
11
- const bytes = await collectBytes(content);
12
- const workbook = new ExcelJS.Workbook();
13
- const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
14
- await workbook.xlsx.load(arrayBuffer);
15
- const sections = [];
16
- workbook.eachSheet((sheet) => {
17
- const rows = [];
18
- sheet.eachRow((row) => {
19
- const cells = [];
20
- row.eachCell({ includeEmpty: true }, (cell) => {
21
- try {
22
- const v = cell.value;
23
- cells.push(v == null ? "" : String(v));
24
- } catch {
25
- cells.push("");
26
- }
26
+ function createXlsxExtractor(limits = {}) {
27
+ const { maxBytes = DEFAULT_MAX_BYTES, maxUncompressedBytes = DEFAULT_MAX_UNCOMPRESSED_BYTES } = limits;
28
+ return async (content) => {
29
+ const bytes = await collectBytes(content, { maxBytes });
30
+ assertZipUncompressedSize(bytes, maxUncompressedBytes);
31
+ const workbook = new ExcelJS.Workbook();
32
+ const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
33
+ await workbook.xlsx.load(arrayBuffer);
34
+ const sections = [];
35
+ workbook.eachSheet((sheet) => {
36
+ const rows = [];
37
+ sheet.eachRow((row) => {
38
+ const cells = [];
39
+ row.eachCell({ includeEmpty: true }, (cell) => {
40
+ try {
41
+ const v = cell.value;
42
+ cells.push(v == null ? "" : escapeCell(String(v)));
43
+ } catch {
44
+ cells.push("");
45
+ }
46
+ });
47
+ rows.push(cells);
27
48
  });
28
- rows.push(cells);
29
- });
30
- if (rows.length === 0) return;
31
- const colCount = Math.max(...rows.map((r) => r.length));
32
- const [headers, ...dataRows] = rows.map((r) => {
33
- while (r.length < colCount) r.push("");
34
- return r;
49
+ if (rows.length === 0) return;
50
+ const colCount = maxColumnCount(rows);
51
+ for (const r of rows) while (r.length < colCount) r.push("");
52
+ const [headers, ...dataRows] = rows;
53
+ if (!headers) return;
54
+ const table = [
55
+ `| ${headers.join(" | ")} |`,
56
+ `| ${headers.map(() => "---").join(" | ")} |`,
57
+ ...dataRows.map((r) => `| ${r.join(" | ")} |`)
58
+ ].join("\n");
59
+ sections.push(`## ${sheet.name}\n\n${table}`);
35
60
  });
36
- if (!headers) return;
37
- const table = [
38
- `| ${headers.join(" | ")} |`,
39
- `| ${headers.map(() => "---").join(" | ")} |`,
40
- ...dataRows.map((r) => `| ${r.join(" | ")} |`)
41
- ].join("\n");
42
- sections.push(`## ${sheet.name}\n\n${table}`);
43
- });
44
- return sections.join("\n\n");
45
- };
61
+ return sections.join("\n\n");
62
+ };
63
+ }
64
+ /** XLSX extractor with the default limits. */
65
+ const xlsxExtractor = createXlsxExtractor();
46
66
  //#endregion
47
- export { xlsxExtractor };
67
+ export { createXlsxExtractor, escapeCell, maxColumnCount, xlsxExtractor };
48
68
 
49
69
  //# sourceMappingURL=xlsx-extractor.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"xlsx-extractor.js","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"sourcesContent":["import ExcelJS from \"exceljs\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport type { ContentExtractor } from \"../types.js\";\n\n/**\n * Converts Excel spreadsheets into markdown tables so the content\n * is searchable and consumable by text-based pipelines. Each worksheet\n * becomes a separate section with a level-2 heading. Empty sheets are\n * skipped to avoid noise.\n */\nexport const xlsxExtractor: ContentExtractor = async (content) => {\n const bytes = await collectBytes(content);\n const workbook = new ExcelJS.Workbook();\n const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);\n await workbook.xlsx.load(arrayBuffer as ArrayBuffer);\n\n const sections: string[] = [];\n\n workbook.eachSheet((sheet) => {\n const rows: string[][] = [];\n\n sheet.eachRow((row) => {\n const cells: string[] = [];\n row.eachCell({ includeEmpty: true }, (cell) => {\n try {\n const v = cell.value;\n cells.push(v == null ? \"\" : String(v));\n } catch {\n cells.push(\"\");\n }\n });\n rows.push(cells);\n });\n\n if (rows.length === 0) return;\n\n // Normalize column count to the widest row\n const colCount = Math.max(...rows.map((r) => r.length));\n const padded = rows.map((r) => {\n while (r.length < colCount) r.push(\"\");\n return r;\n });\n\n // First row as headers, rest as data\n const [headers, ...dataRows] = padded;\n if (!headers) return;\n\n const headerLine = `| ${headers.join(\" | \")} |`;\n const separator = `| ${headers.map(() => \"---\").join(\" | \")} |`;\n const bodyLines = dataRows.map((r) => `| ${r.join(\" | \")} |`);\n\n const table = [headerLine, separator, ...bodyLines].join(\"\\n\");\n sections.push(`## ${sheet.name}\\n\\n${table}`);\n });\n\n return sections.join(\"\\n\\n\");\n};\n"],"mappings":";;;;;;;;;AAUA,MAAa,gBAAkC,OAAO,YAAY;CAChE,MAAM,QAAQ,MAAM,aAAa,OAAO;CACxC,MAAM,WAAW,IAAI,QAAQ,SAAS;CACtC,MAAM,cAAc,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;CAC5F,MAAM,SAAS,KAAK,KAAK,WAA0B;CAEnD,MAAM,WAAqB,CAAC;CAE5B,SAAS,WAAW,UAAU;EAC5B,MAAM,OAAmB,CAAC;EAE1B,MAAM,SAAS,QAAQ;GACrB,MAAM,QAAkB,CAAC;GACzB,IAAI,SAAS,EAAE,cAAc,KAAK,IAAI,SAAS;IAC7C,IAAI;KACF,MAAM,IAAI,KAAK;KACf,MAAM,KAAK,KAAK,OAAO,KAAK,OAAO,CAAC,CAAC;IACvC,QAAQ;KACN,MAAM,KAAK,EAAE;IACf;GACF,CAAC;GACD,KAAK,KAAK,KAAK;EACjB,CAAC;EAED,IAAI,KAAK,WAAW,GAAG;EAGvB,MAAM,WAAW,KAAK,IAAI,GAAG,KAAK,KAAK,MAAM,EAAE,MAAM,CAAC;EAOtD,MAAM,CAAC,SAAS,GAAG,YANJ,KAAK,KAAK,MAAM;GAC7B,OAAO,EAAE,SAAS,UAAU,EAAE,KAAK,EAAE;GACrC,OAAO;EACT,CAGoC;EACpC,IAAI,CAAC,SAAS;EAMd,MAAM,QAAQ;GAAC,KAJS,QAAQ,KAAK,KAAK,EAAE;GAIjB,KAHJ,QAAQ,UAAU,KAAK,CAAC,CAAC,KAAK,KAAK,EAAE;GAGtB,GAFpB,SAAS,KAAK,MAAM,KAAK,EAAE,KAAK,KAAK,EAAE,GAER;EAAC,CAAC,CAAC,KAAK,IAAI;EAC7D,SAAS,KAAK,MAAM,MAAM,KAAK,MAAM,OAAO;CAC9C,CAAC;CAED,OAAO,SAAS,KAAK,MAAM;AAC7B"}
1
+ {"version":3,"file":"xlsx-extractor.js","names":[],"sources":["../../src/extractors/xlsx-extractor.ts"],"sourcesContent":["import ExcelJS from \"exceljs\";\nimport { collectBytes } from \"../collect-bytes.js\";\nimport {\n DEFAULT_MAX_BYTES,\n DEFAULT_MAX_UNCOMPRESSED_BYTES,\n type ZipExtractorLimits,\n} from \"../limits.js\";\nimport type { ContentExtractor } from \"../types.js\";\nimport { assertZipUncompressedSize } from \"../zip-size.js\";\n\n/**\n * Makes a cell value safe inside a Markdown table row: `|` becomes `\\|` so it does not start a\n * new column, and every line break (CR, LF, CRLF) becomes one space so it does not end the row.\n */\nexport function escapeCell(value: string): string {\n return value.replace(/\\|/g, \"\\\\|\").replace(/\\r\\n|\\r|\\n/g, \" \");\n}\n\n/** Width of the widest row. A loop, not `Math.max(...)`, so large sheets do not overflow the stack. */\nexport function maxColumnCount(rows: readonly (readonly unknown[])[]): number {\n let max = 0;\n for (const row of rows) {\n if (row.length > max) max = row.length;\n }\n return max;\n}\n\n/**\n * Converts Excel spreadsheets into markdown tables so the content\n * is searchable and consumable by text-based pipelines. Each worksheet\n * becomes a separate section with a level-2 heading. Empty sheets are\n * skipped to avoid noise. The input size and the archive's declared\n * uncompressed size are checked before exceljs inflates it.\n */\nexport function createXlsxExtractor(limits: ZipExtractorLimits = {}): ContentExtractor {\n const { maxBytes = DEFAULT_MAX_BYTES, maxUncompressedBytes = DEFAULT_MAX_UNCOMPRESSED_BYTES } =\n limits;\n return async (content) => {\n const bytes = await collectBytes(content, { maxBytes });\n assertZipUncompressedSize(bytes, maxUncompressedBytes);\n const workbook = new ExcelJS.Workbook();\n const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);\n await workbook.xlsx.load(arrayBuffer as ArrayBuffer);\n\n const sections: string[] = [];\n\n workbook.eachSheet((sheet) => {\n const rows: string[][] = [];\n\n sheet.eachRow((row) => {\n const cells: string[] = [];\n row.eachCell({ includeEmpty: true }, (cell) => {\n try {\n const v = cell.value;\n cells.push(v == null ? \"\" : escapeCell(String(v)));\n } catch {\n cells.push(\"\");\n }\n });\n rows.push(cells);\n });\n\n if (rows.length === 0) return;\n\n // Normalize column count to the widest row\n const colCount = maxColumnCount(rows);\n for (const r of rows) {\n while (r.length < colCount) r.push(\"\");\n }\n\n // First row as headers, rest as data\n const [headers, ...dataRows] = rows;\n if (!headers) return;\n\n const headerLine = `| ${headers.join(\" | \")} |`;\n const separator = `| ${headers.map(() => \"---\").join(\" | \")} |`;\n const bodyLines = dataRows.map((r) => `| ${r.join(\" | \")} |`);\n\n const table = [headerLine, separator, ...bodyLines].join(\"\\n\");\n sections.push(`## ${sheet.name}\\n\\n${table}`);\n });\n\n return sections.join(\"\\n\\n\");\n };\n}\n\n/** XLSX extractor with the default limits. */\nexport const xlsxExtractor: ContentExtractor = createXlsxExtractor();\n"],"mappings":";;;;;;;;;AAcA,SAAgB,WAAW,OAAuB;CAChD,OAAO,MAAM,QAAQ,OAAO,KAAK,CAAC,CAAC,QAAQ,eAAe,GAAG;AAC/D;;AAGA,SAAgB,eAAe,MAA+C;CAC5E,IAAI,MAAM;CACV,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,SAAS,KAAK,MAAM,IAAI;CAElC,OAAO;AACT;;;;;;;;AASA,SAAgB,oBAAoB,SAA6B,CAAC,GAAqB;CACrF,MAAM,EAAE,WAAW,mBAAmB,uBAAuB,mCAC3D;CACF,OAAO,OAAO,YAAY;EACxB,MAAM,QAAQ,MAAM,aAAa,SAAS,EAAE,SAAS,CAAC;EACtD,0BAA0B,OAAO,oBAAoB;EACrD,MAAM,WAAW,IAAI,QAAQ,SAAS;EACtC,MAAM,cAAc,MAAM,OAAO,MAAM,MAAM,YAAY,MAAM,aAAa,MAAM,UAAU;EAC5F,MAAM,SAAS,KAAK,KAAK,WAA0B;EAEnD,MAAM,WAAqB,CAAC;EAE5B,SAAS,WAAW,UAAU;GAC5B,MAAM,OAAmB,CAAC;GAE1B,MAAM,SAAS,QAAQ;IACrB,MAAM,QAAkB,CAAC;IACzB,IAAI,SAAS,EAAE,cAAc,KAAK,IAAI,SAAS;KAC7C,IAAI;MACF,MAAM,IAAI,KAAK;MACf,MAAM,KAAK,KAAK,OAAO,KAAK,WAAW,OAAO,CAAC,CAAC,CAAC;KACnD,QAAQ;MACN,MAAM,KAAK,EAAE;KACf;IACF,CAAC;IACD,KAAK,KAAK,KAAK;GACjB,CAAC;GAED,IAAI,KAAK,WAAW,GAAG;GAGvB,MAAM,WAAW,eAAe,IAAI;GACpC,KAAK,MAAM,KAAK,MACd,OAAO,EAAE,SAAS,UAAU,EAAE,KAAK,EAAE;GAIvC,MAAM,CAAC,SAAS,GAAG,YAAY;GAC/B,IAAI,CAAC,SAAS;GAMd,MAAM,QAAQ;IAAC,KAJS,QAAQ,KAAK,KAAK,EAAE;IAIjB,KAHJ,QAAQ,UAAU,KAAK,CAAC,CAAC,KAAK,KAAK,EAAE;IAGtB,GAFpB,SAAS,KAAK,MAAM,KAAK,EAAE,KAAK,KAAK,EAAE,GAER;GAAC,CAAC,CAAC,KAAK,IAAI;GAC7D,SAAS,KAAK,MAAM,MAAM,KAAK,MAAM,OAAO;EAC9C,CAAC;EAED,OAAO,SAAS,KAAK,MAAM;CAC7B;AACF;;AAGA,MAAa,gBAAkC,oBAAoB"}
@@ -3,7 +3,9 @@
3
3
  * Normalizes HTML into markdown so downstream consumers get a uniform
4
4
  * text format regardless of whether the source was HTML, DOCX (via
5
5
  * mammoth's HTML output), or any other HTML-producing extractor.
6
- * Uses Turndown with GFM tables and strikethrough support.
6
+ * Uses Turndown with GFM tables and strikethrough support. The content of
7
+ * `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is
8
+ * code or markup, not document text.
7
9
  */
8
10
  export declare function htmlToMarkdown(html: string): string;
9
11
  //#endregion
@@ -1 +1 @@
1
- {"version":3,"file":"html-to-markdown.d.ts","names":[],"sources":["../src/html-to-markdown.ts"],"mappings":";;;;;;;wBAUgB,eAAe"}
1
+ {"version":3,"file":"html-to-markdown.d.ts","names":[],"sources":["../src/html-to-markdown.ts"],"mappings":";;;;;;;;;wBAYgB,eAAe"}
@@ -5,7 +5,9 @@ import TurndownService from "turndown";
5
5
  * Normalizes HTML into markdown so downstream consumers get a uniform
6
6
  * text format regardless of whether the source was HTML, DOCX (via
7
7
  * mammoth's HTML output), or any other HTML-producing extractor.
8
- * Uses Turndown with GFM tables and strikethrough support.
8
+ * Uses Turndown with GFM tables and strikethrough support. The content of
9
+ * `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is
10
+ * code or markup, not document text.
9
11
  */
10
12
  function htmlToMarkdown(html) {
11
13
  const turndownService = new TurndownService({
@@ -17,6 +19,12 @@ function htmlToMarkdown(html) {
17
19
  strongDelimiter: "**"
18
20
  });
19
21
  turndownService.use(gfm);
22
+ turndownService.remove([
23
+ "script",
24
+ "style",
25
+ "noscript",
26
+ "template"
27
+ ]);
20
28
  return turndownService.turndown(html);
21
29
  }
22
30
  //#endregion
@@ -1 +1 @@
1
- {"version":3,"file":"html-to-markdown.js","names":[],"sources":["../src/html-to-markdown.ts"],"sourcesContent":["/// <reference path=\"./turndown-plugin-gfm.d.ts\" />\nimport { gfm } from \"@joplin/turndown-plugin-gfm\";\nimport TurndownService from \"turndown\";\n\n/**\n * Normalizes HTML into markdown so downstream consumers get a uniform\n * text format regardless of whether the source was HTML, DOCX (via\n * mammoth's HTML output), or any other HTML-producing extractor.\n * Uses Turndown with GFM tables and strikethrough support.\n */\nexport function htmlToMarkdown(html: string): string {\n const turndownService = new TurndownService({\n headingStyle: \"atx\",\n hr: \"---\",\n bulletListMarker: \"*\",\n codeBlockStyle: \"fenced\",\n emDelimiter: \"*\",\n strongDelimiter: \"**\",\n });\n turndownService.use(gfm);\n return turndownService.turndown(html);\n}\n"],"mappings":";;;;;;;;;AAUA,SAAgB,eAAe,MAAsB;CACnD,MAAM,kBAAkB,IAAI,gBAAgB;EAC1C,cAAc;EACd,IAAI;EACJ,kBAAkB;EAClB,gBAAgB;EAChB,aAAa;EACb,iBAAiB;CACnB,CAAC;CACD,gBAAgB,IAAI,GAAG;CACvB,OAAO,gBAAgB,SAAS,IAAI;AACtC"}
1
+ {"version":3,"file":"html-to-markdown.js","names":[],"sources":["../src/html-to-markdown.ts"],"sourcesContent":["/// <reference path=\"./turndown-plugin-gfm.d.ts\" />\nimport { gfm } from \"@joplin/turndown-plugin-gfm\";\nimport TurndownService from \"turndown\";\n\n/**\n * Normalizes HTML into markdown so downstream consumers get a uniform\n * text format regardless of whether the source was HTML, DOCX (via\n * mammoth's HTML output), or any other HTML-producing extractor.\n * Uses Turndown with GFM tables and strikethrough support. The content of\n * `<script>`, `<style>`, `<noscript>` and `<template>` is dropped: it is\n * code or markup, not document text.\n */\nexport function htmlToMarkdown(html: string): string {\n const turndownService = new TurndownService({\n headingStyle: \"atx\",\n hr: \"---\",\n bulletListMarker: \"*\",\n codeBlockStyle: \"fenced\",\n emDelimiter: \"*\",\n strongDelimiter: \"**\",\n });\n turndownService.use(gfm);\n turndownService.remove([\"script\", \"style\", \"noscript\", \"template\"]);\n return turndownService.turndown(html);\n}\n"],"mappings":";;;;;;;;;;;AAYA,SAAgB,eAAe,MAAsB;CACnD,MAAM,kBAAkB,IAAI,gBAAgB;EAC1C,cAAc;EACd,IAAI;EACJ,kBAAkB;EAClB,gBAAgB;EAChB,aAAa;EACb,iBAAiB;CACnB,CAAC;CACD,gBAAgB,IAAI,GAAG;CACvB,gBAAgB,OAAO;EAAC;EAAU;EAAS;EAAY;CAAU,CAAC;CAClE,OAAO,gBAAgB,SAAS,IAAI;AACtC"}
package/dist/index.d.ts CHANGED
@@ -1,14 +1,16 @@
1
+ import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorLimits, ZipExtractorLimits } from "./limits.js";
1
2
  import { ContentExtractor, ContentNormalizer } from "./types.js";
2
- import { docxExtractor } from "./extractors/docx-extractor.js";
3
- import { htmlExtractor } from "./extractors/html-extractor.js";
4
- import { markdownExtractor } from "./extractors/markdown-extractor.js";
5
- import { pdfExtractor } from "./extractors/pdf-extractor.js";
6
- import { textExtractor } from "./extractors/text-extractor.js";
7
- import { xlsxExtractor } from "./extractors/xlsx-extractor.js";
3
+ import { createDocxExtractor, docxExtractor } from "./extractors/docx-extractor.js";
4
+ import { createHtmlExtractor, htmlExtractor } from "./extractors/html-extractor.js";
5
+ import { createMarkdownExtractor, markdownExtractor } from "./extractors/markdown-extractor.js";
6
+ import { createPdfExtractor, pdfExtractor } from "./extractors/pdf-extractor.js";
7
+ import { createTextExtractor, textExtractor } from "./extractors/text-extractor.js";
8
+ import { createXlsxExtractor, xlsxExtractor } from "./extractors/xlsx-extractor.js";
8
9
  import { ExtractorRegistry } from "./extractor-registry.js";
9
10
  import { createDefaultRegistry } from "./extractors/index.js";
10
- import { collectBytes, collectText } from "./collect-bytes.js";
11
+ import { CollectOptions, collectBytes, collectText } from "./collect-bytes.js";
11
12
  import { htmlToMarkdown } from "./html-to-markdown.js";
12
- import { markdownToHtml } from "./markdown-to-html.js";
13
+ import { MarkdownToHtmlOptions, markdownToHtml } from "./markdown-to-html.js";
13
14
  import { getMimeType } from "./mime-utils.js";
14
- export { ContentExtractor, ContentNormalizer, ExtractorRegistry, collectBytes, collectText, createDefaultRegistry, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor };
15
+ import { assertZipUncompressedSize, zipUncompressedSize } from "./zip-size.js";
16
+ export { CollectOptions, ContentExtractor, ContentNormalizer, DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorLimits, ExtractorRegistry, MarkdownToHtmlOptions, ZipExtractorLimits, assertZipUncompressedSize, collectBytes, collectText, createDefaultRegistry, createDocxExtractor, createHtmlExtractor, createMarkdownExtractor, createPdfExtractor, createTextExtractor, createXlsxExtractor, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor, zipUncompressedSize };
package/dist/index.js CHANGED
@@ -1,13 +1,15 @@
1
1
  import { collectBytes, collectText } from "./collect-bytes.js";
2
2
  import { htmlToMarkdown } from "./html-to-markdown.js";
3
- import { docxExtractor } from "./extractors/docx-extractor.js";
4
- import { htmlExtractor } from "./extractors/html-extractor.js";
5
- import { markdownExtractor } from "./extractors/markdown-extractor.js";
6
- import { pdfExtractor } from "./extractors/pdf-extractor.js";
7
- import { textExtractor } from "./extractors/text-extractor.js";
8
- import { xlsxExtractor } from "./extractors/xlsx-extractor.js";
3
+ import { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES } from "./limits.js";
4
+ import { assertZipUncompressedSize, zipUncompressedSize } from "./zip-size.js";
5
+ import { createDocxExtractor, docxExtractor } from "./extractors/docx-extractor.js";
6
+ import { createHtmlExtractor, htmlExtractor } from "./extractors/html-extractor.js";
7
+ import { createMarkdownExtractor, markdownExtractor } from "./extractors/markdown-extractor.js";
8
+ import { createPdfExtractor, pdfExtractor } from "./extractors/pdf-extractor.js";
9
+ import { createTextExtractor, textExtractor } from "./extractors/text-extractor.js";
10
+ import { createXlsxExtractor, xlsxExtractor } from "./extractors/xlsx-extractor.js";
9
11
  import { ExtractorRegistry } from "./extractor-registry.js";
10
12
  import { createDefaultRegistry } from "./extractors/index.js";
11
13
  import { markdownToHtml } from "./markdown-to-html.js";
12
14
  import { getMimeType } from "./mime-utils.js";
13
- export { ExtractorRegistry, collectBytes, collectText, createDefaultRegistry, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor };
15
+ export { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES, ExtractorRegistry, assertZipUncompressedSize, collectBytes, collectText, createDefaultRegistry, createDocxExtractor, createHtmlExtractor, createMarkdownExtractor, createPdfExtractor, createTextExtractor, createXlsxExtractor, docxExtractor, getMimeType, htmlExtractor, htmlToMarkdown, markdownExtractor, markdownToHtml, pdfExtractor, textExtractor, xlsxExtractor, zipUncompressedSize };
@@ -0,0 +1,20 @@
1
+ //#region src/limits.d.ts
2
+ /** Default largest input an extractor reads: 100 MiB. */
3
+ export declare const DEFAULT_MAX_BYTES: number;
4
+ /** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */
5
+ export declare const DEFAULT_MAX_UNCOMPRESSED_BYTES: number;
6
+ /** Limits every built-in extractor accepts. Pass `Infinity` to turn a limit off. */
7
+ export interface ExtractorLimits {
8
+ /** Largest input read, in bytes. Default {@link DEFAULT_MAX_BYTES}. */
9
+ maxBytes?: number;
10
+ }
11
+ /** Limits of the ZIP-based extractors (DOCX, DOCM, XLSX). */
12
+ export interface ZipExtractorLimits extends ExtractorLimits {
13
+ /**
14
+ * Largest sum of the uncompressed entry sizes declared in the archive's central directory,
15
+ * checked before the archive is inflated. Default {@link DEFAULT_MAX_UNCOMPRESSED_BYTES}.
16
+ */
17
+ maxUncompressedBytes?: number;
18
+ }
19
+ //#endregion
20
+ //# sourceMappingURL=limits.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"limits.d.ts","names":[],"sources":["../src/limits.ts"],"mappings":";;qBACa;;qBAGA;;iBAGI;;EAEf;;;iBAIe,2BAA2B;;;;;EAK1C"}
package/dist/limits.js ADDED
@@ -0,0 +1,9 @@
1
+ //#region src/limits.ts
2
+ /** Default largest input an extractor reads: 100 MiB. */
3
+ const DEFAULT_MAX_BYTES = 104857600;
4
+ /** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */
5
+ const DEFAULT_MAX_UNCOMPRESSED_BYTES = 524288e3;
6
+ //#endregion
7
+ export { DEFAULT_MAX_BYTES, DEFAULT_MAX_UNCOMPRESSED_BYTES };
8
+
9
+ //# sourceMappingURL=limits.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"limits.js","names":[],"sources":["../src/limits.ts"],"sourcesContent":["/** Default largest input an extractor reads: 100 MiB. */\nexport const DEFAULT_MAX_BYTES = 100 * 1024 * 1024;\n\n/** Default largest total uncompressed size of a DOCX/XLSX archive: 500 MiB. */\nexport const DEFAULT_MAX_UNCOMPRESSED_BYTES = 500 * 1024 * 1024;\n\n/** Limits every built-in extractor accepts. Pass `Infinity` to turn a limit off. */\nexport interface ExtractorLimits {\n /** Largest input read, in bytes. Default {@link DEFAULT_MAX_BYTES}. */\n maxBytes?: number;\n}\n\n/** Limits of the ZIP-based extractors (DOCX, DOCM, XLSX). */\nexport interface ZipExtractorLimits extends ExtractorLimits {\n /**\n * Largest sum of the uncompressed entry sizes declared in the archive's central directory,\n * checked before the archive is inflated. Default {@link DEFAULT_MAX_UNCOMPRESSED_BYTES}.\n */\n maxUncompressedBytes?: number;\n}\n"],"mappings":";;AACA,MAAa,oBAAoB;;AAGjC,MAAa,iCAAiC"}
@@ -1,10 +1,18 @@
1
1
  //#region src/markdown-to-html.d.ts
2
+ export interface MarkdownToHtmlOptions {
3
+ /**
4
+ * Pass raw HTML in the Markdown through to the output. Off by default: raw HTML is escaped,
5
+ * so `<script>` or `<img onerror>` in an uploaded file cannot run when the output is shown.
6
+ * Turn it on only for trusted input, or sanitise the output before rendering it.
7
+ */
8
+ html?: boolean;
9
+ }
2
10
  /**
3
11
  * Provides the reverse of `htmlToMarkdown` for cases where extracted
4
12
  * markdown needs to be rendered back to HTML (e.g. previews, email
5
- * output). Configured with pass-through raw HTML and auto-linking
6
- * so embedded markup and bare URLs survive the round-trip.
13
+ * output). Bare URLs are auto-linked. Raw HTML is escaped unless
14
+ * `options.html` is set.
7
15
  */
8
- export declare function markdownToHtml(markdown: string): string;
16
+ export declare function markdownToHtml(markdown: string, options?: MarkdownToHtmlOptions): string;
9
17
  //#endregion
10
18
  //# sourceMappingURL=markdown-to-html.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"markdown-to-html.d.ts","names":[],"sources":["../src/markdown-to-html.ts"],"mappings":";;;;;;;wBAegB,eAAe"}
1
+ {"version":3,"file":"markdown-to-html.d.ts","names":[],"sources":["../src/markdown-to-html.ts"],"mappings":";iBAMiB;;;;;;EAMf;;;;;;;;wBASc,eAAe,kBAAkB,UAAS"}
@@ -1,20 +1,27 @@
1
1
  import MarkdownIt from "markdown-it";
2
2
  //#region src/markdown-to-html.ts
3
- const md = new MarkdownIt({
4
- html: true,
3
+ const options = {
5
4
  linkify: true,
6
5
  typographer: true,
7
6
  breaks: false
7
+ };
8
+ const escaping = new MarkdownIt({
9
+ ...options,
10
+ html: false
11
+ });
12
+ const passThrough = new MarkdownIt({
13
+ ...options,
14
+ html: true
8
15
  });
9
16
  /**
10
17
  * Provides the reverse of `htmlToMarkdown` for cases where extracted
11
18
  * markdown needs to be rendered back to HTML (e.g. previews, email
12
- * output). Configured with pass-through raw HTML and auto-linking
13
- * so embedded markup and bare URLs survive the round-trip.
19
+ * output). Bare URLs are auto-linked. Raw HTML is escaped unless
20
+ * `options.html` is set.
14
21
  */
15
- function markdownToHtml(markdown) {
22
+ function markdownToHtml(markdown, options = {}) {
16
23
  if (!markdown || markdown.trim() === "") return "";
17
- return md.render(markdown);
24
+ return (options.html ? passThrough : escaping).render(markdown);
18
25
  }
19
26
  //#endregion
20
27
  export { markdownToHtml };
@@ -1 +1 @@
1
- {"version":3,"file":"markdown-to-html.js","names":[],"sources":["../src/markdown-to-html.ts"],"sourcesContent":["import MarkdownIt from \"markdown-it\";\n\nconst md = new MarkdownIt({\n html: true,\n linkify: true,\n typographer: true,\n breaks: false,\n});\n\n/**\n * Provides the reverse of `htmlToMarkdown` for cases where extracted\n * markdown needs to be rendered back to HTML (e.g. previews, email\n * output). Configured with pass-through raw HTML and auto-linking\n * so embedded markup and bare URLs survive the round-trip.\n */\nexport function markdownToHtml(markdown: string): string {\n if (!markdown || markdown.trim() === \"\") {\n return \"\";\n }\n return md.render(markdown);\n}\n"],"mappings":";;AAEA,MAAM,KAAK,IAAI,WAAW;CACxB,MAAM;CACN,SAAS;CACT,aAAa;CACb,QAAQ;AACV,CAAC;;;;;;;AAQD,SAAgB,eAAe,UAA0B;CACvD,IAAI,CAAC,YAAY,SAAS,KAAK,MAAM,IACnC,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ;AAC3B"}
1
+ {"version":3,"file":"markdown-to-html.js","names":[],"sources":["../src/markdown-to-html.ts"],"sourcesContent":["import MarkdownIt from \"markdown-it\";\n\nconst options = { linkify: true, typographer: true, breaks: false } as const;\nconst escaping = new MarkdownIt({ ...options, html: false });\nconst passThrough = new MarkdownIt({ ...options, html: true });\n\nexport interface MarkdownToHtmlOptions {\n /**\n * Pass raw HTML in the Markdown through to the output. Off by default: raw HTML is escaped,\n * so `<script>` or `<img onerror>` in an uploaded file cannot run when the output is shown.\n * Turn it on only for trusted input, or sanitise the output before rendering it.\n */\n html?: boolean;\n}\n\n/**\n * Provides the reverse of `htmlToMarkdown` for cases where extracted\n * markdown needs to be rendered back to HTML (e.g. previews, email\n * output). Bare URLs are auto-linked. Raw HTML is escaped unless\n * `options.html` is set.\n */\nexport function markdownToHtml(markdown: string, options: MarkdownToHtmlOptions = {}): string {\n if (!markdown || markdown.trim() === \"\") {\n return \"\";\n }\n return (options.html ? passThrough : escaping).render(markdown);\n}\n"],"mappings":";;AAEA,MAAM,UAAU;CAAE,SAAS;CAAM,aAAa;CAAM,QAAQ;AAAM;AAClE,MAAM,WAAW,IAAI,WAAW;CAAE,GAAG;CAAS,MAAM;AAAM,CAAC;AAC3D,MAAM,cAAc,IAAI,WAAW;CAAE,GAAG;CAAS,MAAM;AAAK,CAAC;;;;;;;AAiB7D,SAAgB,eAAe,UAAkB,UAAiC,CAAC,GAAW;CAC5F,IAAI,CAAC,YAAY,SAAS,KAAK,MAAM,IACnC,OAAO;CAET,QAAQ,QAAQ,OAAO,cAAc,SAAA,CAAU,OAAO,QAAQ;AAChE"}
@@ -18,9 +18,10 @@ const MIME_MAP = {
18
18
  * Content-Type header and need a MIME type for the registry fallback path.
19
19
  */
20
20
  function getMimeType(path) {
21
- const dotIndex = path.lastIndexOf(".");
21
+ const name = path.slice(path.lastIndexOf("/") + 1);
22
+ const dotIndex = name.lastIndexOf(".");
22
23
  if (dotIndex === -1) return void 0;
23
- const ext = path.slice(dotIndex).toLowerCase();
24
+ const ext = name.slice(dotIndex).toLowerCase();
24
25
  return MIME_MAP[ext];
25
26
  }
26
27
  //#endregion
@@ -1 +1 @@
1
- {"version":3,"file":"mime-utils.js","names":[],"sources":["../src/mime-utils.ts"],"sourcesContent":["const MIME_MAP: Record<string, string> = {\n \".md\": \"text/markdown\",\n \".markdown\": \"text/markdown\",\n \".txt\": \"text/plain\",\n \".html\": \"text/html\",\n \".htm\": \"text/html\",\n \".json\": \"application/json\",\n \".pdf\": \"application/pdf\",\n \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n \".docm\": \"application/vnd.ms-word.document.macroEnabled.12\",\n \".xlsx\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n};\n\n/**\n * Lightweight alternative to a full MIME-sniffing library -- covers only\n * the extensions this package can actually extract, keeping the dependency\n * footprint small. Used by callers who have a filename but no\n * Content-Type header and need a MIME type for the registry fallback path.\n */\nexport function getMimeType(path: string): string | undefined {\n const dotIndex = path.lastIndexOf(\".\");\n if (dotIndex === -1) return undefined;\n const ext = path.slice(dotIndex).toLowerCase();\n return MIME_MAP[ext];\n}\n"],"mappings":";AAAA,MAAM,WAAmC;CACvC,OAAO;CACP,aAAa;CACb,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,SAAS;CACT,SAAS;AACX;;;;;;;AAQA,SAAgB,YAAY,MAAkC;CAC5D,MAAM,WAAW,KAAK,YAAY,GAAG;CACrC,IAAI,aAAa,IAAI,OAAO,KAAA;CAC5B,MAAM,MAAM,KAAK,MAAM,QAAQ,CAAC,CAAC,YAAY;CAC7C,OAAO,SAAS;AAClB"}
1
+ {"version":3,"file":"mime-utils.js","names":[],"sources":["../src/mime-utils.ts"],"sourcesContent":["const MIME_MAP: Record<string, string> = {\n \".md\": \"text/markdown\",\n \".markdown\": \"text/markdown\",\n \".txt\": \"text/plain\",\n \".html\": \"text/html\",\n \".htm\": \"text/html\",\n \".json\": \"application/json\",\n \".pdf\": \"application/pdf\",\n \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n \".docm\": \"application/vnd.ms-word.document.macroEnabled.12\",\n \".xlsx\": \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n};\n\n/**\n * Lightweight alternative to a full MIME-sniffing library -- covers only\n * the extensions this package can actually extract, keeping the dependency\n * footprint small. Used by callers who have a filename but no\n * Content-Type header and need a MIME type for the registry fallback path.\n */\nexport function getMimeType(path: string): string | undefined {\n const name = path.slice(path.lastIndexOf(\"/\") + 1);\n const dotIndex = name.lastIndexOf(\".\");\n if (dotIndex === -1) return undefined;\n const ext = name.slice(dotIndex).toLowerCase();\n return MIME_MAP[ext];\n}\n"],"mappings":";AAAA,MAAM,WAAmC;CACvC,OAAO;CACP,aAAa;CACb,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,SAAS;CACT,SAAS;AACX;;;;;;;AAQA,SAAgB,YAAY,MAAkC;CAC5D,MAAM,OAAO,KAAK,MAAM,KAAK,YAAY,GAAG,IAAI,CAAC;CACjD,MAAM,WAAW,KAAK,YAAY,GAAG;CACrC,IAAI,aAAa,IAAI,OAAO,KAAA;CAC5B,MAAM,MAAM,KAAK,MAAM,QAAQ,CAAC,CAAC,YAAY;CAC7C,OAAO,SAAS;AAClB"}
@@ -0,0 +1,16 @@
1
+ //#region src/zip-size.d.ts
2
+ /**
3
+ * Sums the uncompressed sizes of all entries declared in a ZIP archive's central directory,
4
+ * without inflating anything. Supports ZIP64. Throws when the bytes are not a readable ZIP.
5
+ *
6
+ * The sizes are what the archive claims; a parser that inflates the data still has to check
7
+ * them, which JSZip (used by mammoth and exceljs) does.
8
+ */
9
+ export declare function zipUncompressedSize(bytes: Uint8Array): number;
10
+ /**
11
+ * Throws when the uncompressed size declared by a ZIP archive exceeds `maxBytes`, so a small
12
+ * "zip bomb" is refused before a parser inflates it.
13
+ */
14
+ export declare function assertZipUncompressedSize(bytes: Uint8Array, maxBytes: number): void;
15
+ //#endregion
16
+ //# sourceMappingURL=zip-size.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"zip-size.d.ts","names":[],"sources":["../src/zip-size.ts"],"mappings":";;;;;;;;wBAkBgB,oBAAoB,OAAO;;;;;wBAuF3B,0BAA0B,OAAO,YAAY"}
@@ -0,0 +1,90 @@
1
+ //#region src/zip-size.ts
2
+ const EOCD_SIG = 101010256;
3
+ const ZIP64_LOCATOR_SIG = 117853008;
4
+ const ZIP64_EOCD_SIG = 101075792;
5
+ const CD_ENTRY_SIG = 33639248;
6
+ const EOCD_SIZE = 22;
7
+ const MAX_COMMENT = 65535;
8
+ function fail(message) {
9
+ throw new Error(`Invalid ZIP archive: ${message}`);
10
+ }
11
+ /**
12
+ * Sums the uncompressed sizes of all entries declared in a ZIP archive's central directory,
13
+ * without inflating anything. Supports ZIP64. Throws when the bytes are not a readable ZIP.
14
+ *
15
+ * The sizes are what the archive claims; a parser that inflates the data still has to check
16
+ * them, which JSZip (used by mammoth and exceljs) does.
17
+ */
18
+ function zipUncompressedSize(bytes) {
19
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
20
+ const len = bytes.byteLength;
21
+ let eocd = -1;
22
+ const stop = Math.max(0, len - EOCD_SIZE - MAX_COMMENT);
23
+ for (let i = len - EOCD_SIZE; i >= stop; i--) if (view.getUint32(i, true) === EOCD_SIG) {
24
+ eocd = i;
25
+ break;
26
+ }
27
+ if (eocd < 0) fail("not a ZIP file (no end of central directory record)");
28
+ let cdEnd = eocd;
29
+ let entries = view.getUint16(eocd + 10, true);
30
+ let cdSize = view.getUint32(eocd + 12, true);
31
+ let cdOffset = view.getUint32(eocd + 16, true);
32
+ if (entries === 65535 || cdSize === 4294967295 || cdOffset === 4294967295) {
33
+ const locator = eocd - 20;
34
+ if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIG) {
35
+ const z64 = Number(view.getBigUint64(locator + 8, true));
36
+ if (z64 + 56 > len || view.getUint32(z64, true) !== ZIP64_EOCD_SIG) fail("bad ZIP64 end of central directory record");
37
+ cdEnd = z64;
38
+ entries = Number(view.getBigUint64(z64 + 32, true));
39
+ cdSize = Number(view.getBigUint64(z64 + 40, true));
40
+ cdOffset = Number(view.getBigUint64(z64 + 48, true));
41
+ }
42
+ }
43
+ if (cdOffset + 4 > len || entries > 0 && view.getUint32(cdOffset, true) !== CD_ENTRY_SIG) {
44
+ const shifted = cdEnd - cdSize;
45
+ if (shifted >= 0 && shifted + 4 <= len && view.getUint32(shifted, true) === CD_ENTRY_SIG) cdOffset = shifted;
46
+ }
47
+ if (cdOffset + cdSize > len) fail("central directory lies outside the file");
48
+ let total = 0;
49
+ let pos = cdOffset;
50
+ for (let n = 0; n < entries; n++) {
51
+ if (pos + 46 > len || view.getUint32(pos, true) !== CD_ENTRY_SIG) fail("truncated or corrupt central directory");
52
+ const size32 = view.getUint32(pos + 24, true);
53
+ const nameLen = view.getUint16(pos + 28, true);
54
+ const extraLen = view.getUint16(pos + 30, true);
55
+ const commentLen = view.getUint16(pos + 32, true);
56
+ let size = size32;
57
+ if (size32 === 4294967295) size = readZip64Size(view, pos + 46 + nameLen, extraLen);
58
+ total += size;
59
+ pos += 46 + nameLen + extraLen + commentLen;
60
+ }
61
+ return total;
62
+ }
63
+ /** Reads the 64-bit uncompressed size from an entry's ZIP64 extended information extra field. */
64
+ function readZip64Size(view, start, length) {
65
+ const end = start + length;
66
+ if (end > view.byteLength) fail("truncated or corrupt central directory");
67
+ let p = start;
68
+ while (p + 4 <= end) {
69
+ const id = view.getUint16(p, true);
70
+ const size = view.getUint16(p + 2, true);
71
+ if (id === 1) {
72
+ if (size < 8 || p + 12 > end) fail("bad ZIP64 extra field");
73
+ return Number(view.getBigUint64(p + 4, true));
74
+ }
75
+ p += 4 + size;
76
+ }
77
+ fail("missing ZIP64 extra field");
78
+ }
79
+ /**
80
+ * Throws when the uncompressed size declared by a ZIP archive exceeds `maxBytes`, so a small
81
+ * "zip bomb" is refused before a parser inflates it.
82
+ */
83
+ function assertZipUncompressedSize(bytes, maxBytes) {
84
+ const size = zipUncompressedSize(bytes);
85
+ if (size > maxBytes) throw new Error(`Archive uncompressed size ${size} bytes exceeds the limit of ${maxBytes} bytes`);
86
+ }
87
+ //#endregion
88
+ export { assertZipUncompressedSize, zipUncompressedSize };
89
+
90
+ //# sourceMappingURL=zip-size.js.map