officeparser 6.1.0 → 6.1.1

package/dist/parsers/RtfParser.js

@@ -92,6 +92,12 @@ class SimpleRtfParser {
     index = 0;
     /** The RTF content as a Buffer */
     buffer;
+    /** Current code page for character decoding (default is Windows-1252) */
+    codePage = 1252;
+    /** Cached TextDecoders for different code pages */
+    decoders = {};
+    /** Buffer for consecutive text bytes to handle multi-byte encodings and UTF-8 detection */
+    pendingBytes = [];
     /** Total length of the buffer */
     length;
     /**
@@ -110,12 +116,14 @@ class SimpleRtfParser {
             const currentGroup = stack[stack.length - 1];
             if (char === 0x7B) { // '{'
                 this.index++;
+                this.flushPendingText(currentGroup);
                 const newGroup = { type: 'group', content: [] };
                 currentGroup.content.push(newGroup);
                 stack.push(newGroup);
             }
             else if (char === 0x7D) { // '}'
                 this.index++;
+                this.flushPendingText(currentGroup);
                 if (stack.length > 1) {
                     stack.pop();
                 }
@@ -133,6 +141,7 @@ class SimpleRtfParser {
                 this.parseText(currentGroup);
             }
         }
+        this.flushPendingText(root);
         return root;
     }
     parseControl(group) {
@@ -141,55 +150,23 @@ class SimpleRtfParser {
         const char = this.buffer[this.index];
         // Special control symbols
         if (char === 0x7B || char === 0x7D || char === 0x5C) { // \{ \} \\
-            group.content.push({ type: 'text', value: String.fromCharCode(char) });
+            this.pendingBytes.push(char);
             this.index++;
             return;
         }
         if (char === 0x27) { // \'xx (hex)
             this.index++;
             if (this.index + 1 < this.length) {
-                const hex = this.buffer.toString('utf8', this.index, this.index + 2);
+                const hex = String.fromCharCode(this.buffer[this.index], this.buffer[this.index + 1]);
                 const code = parseInt(hex, 16);
                 if (!isNaN(code)) {
-                    // RTF hex escapes represent bytes in the document's code page (usually Windows-1252)
-                    // Characters 0x80-0x9F in Windows-1252 don't map directly to Unicode
-                    // We need to convert them properly
-                    const windows1252ToUnicode = {
-                        0x80: 0x20AC, // €
-                        0x82: 0x201A, // ‚
-                        0x83: 0x0192, // ƒ
-                        0x84: 0x201E, // „
-                        0x85: 0x2026, // …
-                        0x86: 0x2020, // †
-                        0x87: 0x2021, // ‡
-                        0x88: 0x02C6, // ˆ
-                        0x89: 0x2030, // ‰
-                        0x8A: 0x0160, // Š
-                        0x8B: 0x2039, // ‹
-                        0x8C: 0x0152, // Œ
-                        0x8E: 0x017D, // Ž
-                        0x91: 0x2018, // '
-                        0x92: 0x2019, // '
-                        0x93: 0x201C, // "
-                        0x94: 0x201D, // "
-                        0x95: 0x2022, // •
-                        0x96: 0x2013, // –
-                        0x97: 0x2014, // —
-                        0x98: 0x02DC, // ˜
-                        0x99: 0x2122, // ™
-                        0x9A: 0x0161, // š
-                        0x9B: 0x203A, // ›
-                        0x9C: 0x0153, // œ
-                        0x9E: 0x017E, // ž
-                        0x9F: 0x0178 // Ÿ
-                    };
-                    const unicodeCode = windows1252ToUnicode[code] || code;
-                    group.content.push({ type: 'text', value: String.fromCharCode(unicodeCode) });
+                    this.pendingBytes.push(code);
                 }
                 this.index += 2;
             }
             return;
         }
+        this.flushPendingText(group);
         if (char === 0x2A) { // \* (ignorable destination)
             // We treat this as a control word named '*'
             group.content.push({ type: 'control', value: '*' });
@@ -231,7 +208,7 @@ class SimpleRtfParser {
             param = parseInt(paramStr, 10);
         }
         // Space after control word is consumed
-        if (this.index < this.length && this.buffer[this.index] === 0x20) {
+        if (name !== '' && this.index < this.length && this.buffer[this.index] === 0x20) {
             this.index++;
         }
         // Handle \binN
@@ -241,6 +218,22 @@ class SimpleRtfParser {
             // \binN is not added to content as we want to ignore it
             return;
         }
+        // Handle encoding control words
+        if (name === 'ansicpg' && param !== undefined) {
+            this.codePage = param;
+        }
+        else if (name === 'ansi') {
+            this.codePage = 1252;
+        }
+        else if (name === 'mac') {
+            this.codePage = 10000;
+        }
+        else if (name === 'pc') {
+            this.codePage = 437;
+        }
+        else if (name === 'pca') {
+            this.codePage = 850;
+        }
         group.content.push({ type: 'control', value: name, param });
         // If this is the first control word in the group, it might be the destination
         if (group.content.length === 1 && group.type === 'group') {
@@ -252,19 +245,91 @@ class SimpleRtfParser {
         }
     }
     parseText(group) {
-        let text = '';
         while (this.index < this.length) {
             const char = this.buffer[this.index];
+            if (char === undefined)
+                break;
             if (char === 0x7B || char === 0x7D || char === 0x5C || char === 0x0D || char === 0x0A) {
                 break;
             }
-            // Basic ASCII text.
-            text += String.fromCharCode(char);
+            this.pendingBytes.push(char);
             this.index++;
         }
-        if (text.length > 0) {
-            group.content.push({ type: 'text', value: text });
+    }
+    /**
+     * Flushes the pending bytes buffer as a text node to the current group.
+     * @param group The group to append the text node to
+     */
+    flushPendingText(group) {
+        if (this.pendingBytes.length > 0) {
+            group.content.push({ type: 'text', value: this.decodeBytes(this.pendingBytes, this.codePage) });
+            this.pendingBytes = [];
+        }
+    }
+    /**
+     * Decodes a byte array using a "UTF-8 first" strategy.
+     * If the bytes form valid UTF-8 and contain non-ASCII characters, UTF-8 is preferred.
+     * Otherwise, falls back to the specified code page.
+     * @param bytes The bytes to decode
+     * @param codePage The RTF code page ID
+     * @returns The decoded string
+     */
+    decodeBytes(bytes, codePage) {
+        const uint8 = new Uint8Array(bytes);
+        // Try UTF-8 first if there are any non-ASCII bytes.
+        // Many modern RTF generators (like calibre or web-based tools) dump UTF-8 bytes 
+        // into the RTF even if the header claims a different code page.
+        if (bytes.some(b => b > 127)) {
+            try {
+                // Use fatal: true to ensure we fall back on invalid UTF-8 sequences
+                const utf8Decoder = new TextDecoder('utf-8', { fatal: true });
+                return utf8Decoder.decode(uint8);
+            }
+            catch (e) {
+                // Not valid UTF-8, continue to code page fallback
+            }
+        }
+        // Fallback to specified code page
+        if (!this.decoders[codePage]) {
+            let encoding = `windows-${codePage}`;
+            if (codePage === 10000)
+                encoding = 'macintosh';
+            else if (codePage === 437)
+                encoding = 'ibm437';
+            else if (codePage === 850)
+                encoding = 'ibm850';
+            try {
+                this.decoders[codePage] = new TextDecoder(encoding);
+            }
+            catch (e) {
+                if (codePage !== 1252) {
+                    try {
+                        this.decoders[codePage] = new TextDecoder('windows-1252');
+                    }
+                    catch (e2) {
+                        return String.fromCharCode(...bytes);
+                    }
+                }
+                else {
+                    return String.fromCharCode(...bytes);
+                }
+            }
         }
+        let result = this.decoders[codePage].decode(uint8);
+        // Safety override for Windows-1252 0x80-0x9F range if TextDecoder behaves like Latin-1.
+        // We replace control characters in the decoded string with their proper 1252 equivalents.
+        if (codePage === 1252 && /[\u0080-\u009F]/.test(result)) {
+            const map = {
+                '\u0080': '€', '\u0082': '‚', '\u0083': 'ƒ', '\u0084': '„', '\u0085': '…',
+                '\u0086': '†', '\u0087': '‡', '\u0088': 'ˆ', '\u0089': '‰', '\u008A': 'Š',
+                '\u008B': '‹', '\u008C': 'Œ', '\u008E': 'Ž', '\u0091': '‘', '\u0092': '’',
+                '\u0093': '“', '\u0094': '”', '\u0095': '•', '\u0096': '–', '\u0097': '—',
+                '\u0098': '˜', '\u0099': '™', '\u009A': 'š', '\u009B': '›', '\u009C': 'œ',
+                '\u009E': 'ž', '\u009F': 'Ÿ'
+            };
+            return result.replace(/[\u0080-\u009F]/g, m => map[m] || m);
+        }
+        return result;
     }
 }
 exports.SimpleRtfParser = SimpleRtfParser;

package/dist/parsers/WordParser.d.ts

@@ -39,6 +39,7 @@
  * - `<w:p>` - Paragraph
  * - `<w:r>` - Run (contiguous text with same formatting)
  * - `<w:t>` - Text content
+ * - `<w:br>` - Line or page break
  * - `<w:b>`, `<w:i>`, `<w:u>` - Bold, italic, underline
  * - `<w:pStyle>` - Paragraph style (for headings)
  * - `<w:numPr>` - List numbering properties

package/dist/parsers/WordParser.js

@@ -40,6 +40,7 @@
  * - `<w:p>` - Paragraph
  * - `<w:r>` - Run (contiguous text with same formatting)
  * - `<w:t>` - Text content
+ * - `<w:br>` - Line or page break
  * - `<w:b>`, `<w:i>`, `<w:u>` - Bold, italic, underline
  * - `<w:pStyle>` - Paragraph style (for headings)
  * - `<w:numPr>` - List numbering properties
@@ -132,7 +133,7 @@ const parseWord = async (buffer, config) => {
         // Font size
         const szMatch = rPrString.match(/<w:sz w:val="(\d+)"/);
         if (szMatch)
-            formatting.size = (parseInt(szMatch[1]) / 2).toString() + 'pt';
+            formatting.size = (parseInt(szMatch[1], 10) / 2).toString() + 'pt';
         // Color
         const colorMatch = rPrString.match(/<w:color w:val="([^"]+)"/);
         if (colorMatch && colorMatch[1] !== 'auto')
@@ -172,6 +173,27 @@ const parseWord = async (buffer, config) => {
         }
         return formatting;
     };
+    // Helper to extract indentation from paragraph properties XML string
+    const extractIndentationFromXml = (pPr) => {
+        const ind = (0, xmlUtils_js_1.getFirstElementByTagName)(pPr, "w:ind");
+        if (ind) {
+            const indentation = {};
+            const left = ind.getAttribute("w:left") || ind.getAttribute("w:start");
+            const right = ind.getAttribute("w:right") || ind.getAttribute("w:end");
+            const firstLine = ind.getAttribute("w:firstLine");
+            const hanging = ind.getAttribute("w:hanging");
+            if (left)
+                indentation.left = parseInt(left, 10);
+            if (right)
+                indentation.right = parseInt(right, 10);
+            if (firstLine)
+                indentation.firstLine = parseInt(firstLine, 10);
+            if (hanging)
+                indentation.hanging = parseInt(hanging, 10);
+            return Object.keys(indentation).length > 0 ? indentation : undefined;
+        }
+        return undefined;
+    };
     const files = await (0, zipUtils_js_1.extractFiles)(buffer, x => !!x.match(documentFileRegex) ||
         !!x.match(footnotesFileRegex) ||
         !!x.match(endnotesFileRegex) ||
@@ -257,6 +279,7 @@ const parseWord = async (buffer, config) => {
                 const formatting = rPr ? extractFormattingFromXml(rPr) : {};
                 let alignment = undefined;
                 let backgroundColor = undefined;
+                let paragraphIndentation = undefined;
                 if (pPr) {
                     const jc = (0, xmlUtils_js_1.getFirstElementByTagName)(pPr, "w:jc");
                     if (jc) {
@@ -271,8 +294,11 @@ const parseWord = async (buffer, config) => {
                         if (fill && fill !== 'auto')
                             backgroundColor = '#' + fill;
                     }
+                    const ind = extractIndentationFromXml(pPr);
+                    if (ind)
+                        paragraphIndentation = ind;
                 }
-                styleMap[styleId] = { formatting, alignment, backgroundColor };
+                styleMap[styleId] = { formatting, alignment, backgroundColor, paragraphIndentation };
             }
         }
     }
@@ -339,6 +365,14 @@ const parseWord = async (buffer, config) => {
                 }
             }
         }
+        // Extract Indentation
+        let paraIndentation = styleProps.paragraphIndentation;
+        if (pPr) {
+            const ind = extractIndentationFromXml(pPr);
+            if (ind) {
+                paraIndentation = { ...paraIndentation, ...ind };
+            }
+        }
         // Extract Paragraph Background
         let paraBackgroundColor = styleProps.backgroundColor;
         if (pPr) {
@@ -406,26 +440,71 @@ const parseWord = async (buffer, config) => {
                 if (!formatting.backgroundColor && paraBackgroundColor) {
                     formatting.backgroundColor = paraBackgroundColor;
                 }
-                // Text content
-                const tNodes = (0, xmlUtils_js_1.getElementsByTagName)(runNode, "w:t");
-                for (const tNode of tNodes) {
-                    const tContent = tNode.textContent || '';
-                    text += tContent;
-                    const textNode = {
-                        type: 'text',
-                        text: tContent,
-                        formatting: formatting
-                    };
-                    if (config.includeRawContent) {
-                        textNode.rawContent = (0, xmlUtils_js_1.getRawContent)(tNode, documentContent, config);
+                for (const child of runNode.childNodes) {
+                    if (!(0, xmlUtils_js_1.isElement)(child))
+                        continue;
+                    // also handle unprefixed version (mirroring the behaviour of getElementsByTagName)
+                    // Text content
+                    if (child.tagName === "w:t" || child.tagName === "t") {
+                        const tNode = child;
+                        const tContent = tNode.textContent || '';
+                        text += tContent;
+                        const textNode = {
+                            type: 'text',
+                            text: tContent,
+                            formatting: formatting
+                        };
+                        if (config.includeRawContent) {
+                            textNode.rawContent = (0, xmlUtils_js_1.getRawContent)(tNode, documentContent, config);
+                        }
+                        // Always set a style: run style > paragraph style > detected default
+                        // Use detected default style for international compatibility
+                        const nodeStyle = rStyleVal || pStyleVal || defaultParaStyleId;
+                        if (nodeStyle) {
+                            textNode.metadata = { style: nodeStyle };
+                        }
+                        children.push(textNode);
                     }
-                    // Always set a style: run style > paragraph style > detected default
-                    // Use detected default style for international compatibility
-                    const nodeStyle = rStyleVal || pStyleVal || defaultParaStyleId;
-                    if (nodeStyle) {
-                        textNode.metadata = { style: nodeStyle };
+                    // Break nodes
+                    else if (config.includeBreakNodes &&
+                        (child.tagName === "w:br"
+                            || child.tagName === "br"
+                            || child.tagName === "w:cr"
+                            || child.tagName === "cr")) {
+                        const brNode = child;
+                        let breakType = 'textWrapping';
+                        if (child.tagName === "w:cr" || child.tagName === "cr") {
+                            breakType = 'carriageReturn';
+                        }
+                        else {
+                            const nodeBreakType = brNode.getAttribute("w:type") || brNode.getAttribute("type");
+                            if (nodeBreakType !== null) {
+                                breakType = nodeBreakType;
+                            }
+                        }
+                        let breakClear = undefined;
+                        if (breakType === 'textWrapping' && brNode.getAttribute("w:clear") !== null) {
+                            breakClear = brNode.getAttribute("w:clear");
+                        }
+                        const breakNode = {
+                            type: 'break',
+                            metadata: { breakType, clear: breakClear }
+                        };
+                        if (config.includeRawContent) {
+                            breakNode.rawContent = (0, xmlUtils_js_1.getRawContent)(brNode, documentContent, config);
+                        }
+                        children.push(breakNode);
+                    }
+                    else if (config.includeBreakNodes && (child.tagName === "w:lastRenderedPageBreak" || child.tagName === "lastRenderedPageBreak")) {
+                        const breakNode = {
+                            type: 'break',
+                            metadata: { breakType: 'lastRenderedPage' }
+                        };
+                        if (config.includeRawContent) {
+                            breakNode.rawContent = (0, xmlUtils_js_1.getRawContent)(child, documentContent, config);
+                        }
+                        children.push(breakNode);
                     }
-                    children.push(textNode);
                 }
                 // Images/Drawings
                 if (config.extractAttachments) {
@@ -557,7 +636,7 @@ const parseWord = async (buffer, config) => {
             const numIdNode = (0, xmlUtils_js_1.getFirstElementByTagName)(numPr, "w:numId");
             const ilvlNode = (0, xmlUtils_js_1.getFirstElementByTagName)(numPr, "w:ilvl");
             const numId = numIdNode ? numIdNode.getAttribute("w:val") || '0' : '0';
-            const ilvl = ilvlNode ? parseInt(ilvlNode.getAttribute("w:val") || '0') : 0;
+            const ilvl = ilvlNode ? parseInt(ilvlNode.getAttribute("w:val") || '0', 10) : 0;
             let listType = 'ordered';
             let itemIndex = 0;
             if (numId && numberingMap[numId]) {
@@ -591,6 +670,7 @@ const parseWord = async (buffer, config) => {
                 metadata: {
                     listType,
                     indentation: ilvl,
+                    paragraphIndentation: paraIndentation,
                     alignment: (alignment || 'left'),
                     listId: numId,
                     itemIndex: itemIndex,
@@ -602,12 +682,12 @@ const parseWord = async (buffer, config) => {
             return listNode;
         }
         else if (isHeading) {
-            const level = pStyleVal ? parseInt(pStyleVal.replace("Heading", "")) || 1 : 1;
+            const level = pStyleVal ? parseInt(pStyleVal.replace("Heading", ""), 10) || 1 : 1;
             const headingNode = {
                 type: 'heading',
                 text: text,
                 children: children,
-                metadata: { level, alignment, style: pStyleVal ?? undefined }
+                metadata: { level, alignment, paragraphIndentation: paraIndentation, style: pStyleVal ?? undefined }
             };
             if (config.includeRawContent)
                 headingNode.rawContent = (0, xmlUtils_js_1.getRawContent)(pNode, documentContent, config);
@@ -618,7 +698,7 @@ const parseWord = async (buffer, config) => {
                 type: 'paragraph',
                 text: text,
                 children: children,
-                metadata: { alignment, style: pStyleVal ?? undefined }
+                metadata: { alignment, paragraphIndentation: paraIndentation, style: pStyleVal ?? undefined }
             };
             if (config.includeRawContent)
                 paraNode.rawContent = (0, xmlUtils_js_1.getRawContent)(pNode, documentContent, config);
@@ -784,6 +864,9 @@ const parseWord = async (buffer, config) => {
                 if (node.children) {
                     t += node.children.map(getText).filter(t => t != '').join(!node.children[0]?.children ? '' : config.newlineDelimiter ?? '\n');
                 }
+                else if (node.type === 'break') {
+                    t += config.newlineDelimiter ?? '\n';
+                }
                 else
                     t += node.text || '';
                 return t;