officeparser 4.1.1 → 4.2.0

package/README.md

@@ -13,6 +13,8 @@ A Node.js library to parse text out of any office file.
 
 
 #### Update
+* 2024/10/15 - Fixed erroring out while deleting temp files when multiple worker threads make parallel executions resulting in same file name for multiple files. Fixed erroring out when multiple executions are made without waiting for the previous execution to finish which resulted in deleting the file from other execution. Upgraded dependencies.
+* 2024/10/13 - Fixed parsing text from xlsx files which contain no shared strings file and files which have inlineStr based strings.
 * 2024/05/06 - Replaced pdf parsing support from pdf-parse library to natively building it using pdf.js library from Mozilla by analyzing its output. Added pdfjs-dist build as a local library.
 * 2023/11/25 - Fixed error catching when an error occurs within the parsing of a file, especially after decompressing it. Also fixed the problem with parallel parsing of files as we were using only timestamp in file names.
 * 2023/10/24 - Revamped content parsing code. Fixed order of content in files, especially in word files where table information would always land up at the end of the text. Added config object as argument for parseOffice which can be used to set new line delimiter and multiple other configurations. Added support for parsing pdf files using the popular npm library pdf-parse. Removed support for individual file parsing functions.

package/officeParser.js

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 
-const decompress    = require('decompress');
-const fs            = require('fs');
-const rimraf        = require('rimraf');
-const fileType      = require('file-type');
-const pdfjs         = require('./pdfjs-dist-build/pdf.js');
-const { DOMParser } = require('@xmldom/xmldom');
+const decompress     = require('decompress');
+const fs             = require('fs');
+const { rimrafSync } = require('rimraf');
+const fileType       = require('file-type');
+const pdfjs          = require('./pdfjs-dist-build/pdf.js');
+const { DOMParser }  = require('@xmldom/xmldom');
 
 /** Header for error messages */
 const ERRORHEADER = "[OfficeParser]: ";
@@ -126,6 +126,7 @@ function parsePowerPoint(filepath, callback, config) {
     // Files regex that hold our content of interest
     const allFilesRegex = /ppt\/(notesSlides|slides)\/(notesSlide|slide)\d+.xml/g;
     const slidesRegex   = /ppt\/slides\/slide\d+.xml/g;
+    const slideNumberRegex = /lide(\d+)\.xml/;
 
     /** The decompress location which contains the filename in it */
     const decompressLocation = `${config.tempFilesLocation}/${filepath.split("/").pop()}`;
@@ -134,6 +135,17 @@ function parsePowerPoint(filepath, callback, config) {
         { filter: x => x.path.match(config.ignoreNotes ? slidesRegex : allFilesRegex) }
     )
     .then(files => {
+        // Sort files by slide number and their notes (if any).
+        files.sort((a, b) => {
+            const matchedANumber = parseInt(a.path.match(slideNumberRegex)?.at(1), 10);
+            const matchedBNumber = parseInt(b.path.match(slideNumberRegex)?.at(1), 10);
+
+            const aNumber = isNaN(matchedANumber) ? Infinity : matchedANumber;
+            const bNumber = isNaN(matchedBNumber) ? Infinity : matchedBNumber;
+
+            return aNumber - bNumber || Number(a.path.includes('notes')) - Number(b.path.includes('notes'));
+        });
+
         // Verify if atleast the slides xml files exist in the extracted files list.
         if (files.length == 0 || !files.map(file => file.path).some(filename => filename.match(slidesRegex)))
             throw ERRORMSG.fileCorrupted(filepath);
@@ -218,18 +230,45 @@ function parseExcel(filepath, callback, config) {
     })
     // ********************************** excel xml files explanation ***************************************
     // Structure of xmlContent of an excel file is a bit complex.
-    // We have a sharedStrings.xml file which has strings inside t tags
+    // We usually have a sharedStrings.xml file which has strings inside t tags
+    // However, this file is not necessary to be present. It is sometimes absent if the file has no shared strings indices represented in v nodes.
     // Each sheet has an individual sheet xml file which has numbers in v tags (probably value) inside c tags (probably cell)
     // Each value of v tag is to be used as it is if the "t" attribute (probably type) of c tag is not "s" (probably shared string)
     // If the "t" attribute of c tag is "s", then we use the value to select value from sharedStrings array with the value as its index.
+    // However, if the "t" attribute of c tag is "inlineStr", strings can be inline inside "is"(probably inside String) > "t".
+    // We extract either the inline strings or use the value to get numbers of text from shared strings.
     // Drawing files contain all text for each drawing and have text nodes in a:t and paragraph nodes in a:p.
     // ******************************************************************************************************
     .then(xmlContentFilesObject => {
         /** Store all the text content to respond */
         let responseText = [];
 
-        /** Find text nodes with t tags in sharedStrings xml file */
-        const sharedStringsXmlTNodesList = parseString(xmlContentFilesObject.sharedStringsFile).getElementsByTagName("t");
+        /** Function to check if the given c node is a valid inline string node. */
+        function isValidInlineStringCNode(cNode) {
+            // Initial check to see if the passed node is a cNode
+            if (cNode.tagName.toLowerCase() != 'c')
+                return false;
+            if (cNode.getAttribute("t") != 'inlineStr')
+                return false;
+            const childNodesNamedIs = cNode.getElementsByTagName('is');
+            if (childNodesNamedIs.length != 1)
+                return false;
+            const childNodesNamedT = childNodesNamedIs[0].getElementsByTagName('t');
+            if (childNodesNamedT.length != 1)
+                return false;
+            return childNodesNamedT[0].childNodes[0] && childNodesNamedT[0].childNodes[0].nodeValue != '';
+        }
+
+        /** Function to check if the given c node has a valid v node */
+        function hasValidVNodeInCNode(cNode) {
+            return cNode.getElementsByTagName("v")[0]
+                && cNode.getElementsByTagName("v")[0].childNodes[0]
+                && cNode.getElementsByTagName("v")[0].childNodes[0].nodeValue != ''
+        }
+
+        /** Find text nodes with t tags in sharedStrings xml file. If the sharedStringsFile is not present, we return an empty array. */
+        const sharedStringsXmlTNodesList = xmlContentFilesObject.sharedStringsFile != undefined ? parseString(xmlContentFilesObject.sharedStringsFile).getElementsByTagName("t")
+                                                                                                : [];
         /** Create shared string array. This will be used as a map to get strings from within sheet files. */
         const sharedStrings = Array.from(sharedStringsXmlTNodesList)
                                 .map(tNode => tNode.childNodes[0]?.nodeValue ?? '');
@@ -238,25 +277,33 @@ function parseExcel(filepath, callback, config) {
         xmlContentFilesObject.sheetFiles.forEach(sheetXmlContent => {
             /** Find text nodes with c tags in sharedStrings xml file */
             const sheetsXmlCNodesList = parseString(sheetXmlContent).getElementsByTagName("c");
-            // Traverse through the nodes list and fill responseText with either the number value in its v node or find a mapped string from sharedStrings.
+            // Traverse through the nodes list and fill responseText with either the number value in its v node or find a mapped string from sharedStrings or an inline string.
             responseText.push(
                 Array.from(sheetsXmlCNodesList)
-                    // Filter c nodes than do not have any valid v nodes
-                    .filter(cNode => cNode.getElementsByTagName("v")[0]
-                                        && cNode.getElementsByTagName("v")[0].childNodes[0]
-                                        && cNode.getElementsByTagName("v")[0].childNodes[0].nodeValue)
+                    // Filter out invalid c nodes
+                    .filter(cNode => isValidInlineStringCNode(cNode) || hasValidVNodeInCNode(cNode))
                     .map(cNode => {
-                        /** Flag whether this node's value represents a string index */
-                        const isString = cNode.getAttribute("t") == "s";
-                        /** Find value nodes represented by v tags */
-                        const value = cNode.getElementsByTagName("v")[0].childNodes[0].nodeValue;
-                        // Validate text
-                        if (isString && value >= sharedStrings.length)
-                            throw ERRORMSG.fileCorrupted(filepath);
-
-                        return isString
-                                ? sharedStrings[value]
-                                : value;
+                        // Processing if this is a valid inline string c node.
+                        if (isValidInlineStringCNode(cNode))
+                            return cNode.getElementsByTagName('is')[0].getElementsByTagName('t')[0].childNodes[0].nodeValue;
+
+                        // Processing if this c node has a valid v node.
+                        if (hasValidVNodeInCNode(cNode)) {
+                            /** Flag whether this node's value represents an index in the shared string array */
+                            const isIndexInSharedStrings = cNode.getAttribute("t") == "s";
+                            /** Find value nodes represented by v tags */
+                            const value = cNode.getElementsByTagName("v")[0].childNodes[0].nodeValue;
+                            // Validate text
+                            if (isIndexInSharedStrings && value >= sharedStrings.length)
+                                throw ERRORMSG.fileCorrupted(filepath);
+
+                            return isIndexInSharedStrings
+                                    ? sharedStrings[value]
+                                    : value;
+                        }
+                        // TODO: Add debug asserts for if we reach here which would mean we are filtering more items than we are processing.
+                        // Not the case now but it could happen and it is better to be safe.
+                        return '';
                     })
                     // Join each cell text within a sheet with a space.
                     .join(config.newlineDelimiter ?? "\n")
@@ -496,7 +543,7 @@ function parseOffice(file, callback, config = {}) {
         }
 
         // create temp file subdirectory if it does not exist
-        fs.mkdirSync(`${internalConfig.tempFilesLocation}/tempfiles`, { recursive: true });
+        fs.mkdirSync(getTempFilesDirectory(internalConfig.tempFilesLocation), { recursive: true });
 
         // Check if buffer
         if (Buffer.isBuffer(file)) {
@@ -505,11 +552,11 @@ function parseOffice(file, callback, config = {}) {
                 .then(data =>
                 {
                     // temp file name
-                    const newfilepath = getNewFileName(internalConfig.tempFilesLocation, data.ext.toLowerCase());
+                    const newFileName = getNewFileName(data.ext.toLowerCase());
                     // write new file
-                    fs.writeFileSync(newfilepath, file);
+                    fs.writeFileSync(getFilePath(getTempFilesDirectory(internalConfig.tempFilesLocation), newFileName), file);
                     // resolve promise
-                    res(newfilepath);
+                    res(newFileName);
                 })
                 .catch(() => rej(ERRORMSG.improperBuffers));
             return;
@@ -522,16 +569,18 @@ function parseOffice(file, callback, config = {}) {
             throw ERRORMSG.fileDoesNotExist(file);
 
         // temp file name
-        const newfilepath = getNewFileName(internalConfig.tempFilesLocation, file.split(".").pop().toLowerCase());
+        const newFileName = getNewFileName(file.split(".").pop().toLowerCase());
         // Copy the file into a temp location with the temp name
-        fs.copyFileSync(file, newfilepath)
+        fs.copyFileSync(file, getFilePath(getTempFilesDirectory(internalConfig.tempFilesLocation), newFileName))
         // resolve promise
-        res(newfilepath);
+        res(newFileName);
     });
 
     // Process filePreparedPromise resolution.
     filePreparedPromise
-        .then(filepath => {
+        .then(filename => {
+            // The file path
+            const filepath = getFilePath(getTempFilesDirectory(internalConfig.tempFilesLocation), filename);
             // File extension. Already in lowercase when we prepared the temp file above.
             const extension = filepath.split(".").pop();
 
@@ -562,9 +611,33 @@ function parseOffice(file, callback, config = {}) {
             /** Internal callback function that calls the user's callback function passed in argument and removes the temp files if required */
             function internalCallback(data, err) {
                 // Check if we need to preserve unzipped content files or delete them.
-                if (!internalConfig.preserveTempFiles)
-                    // Delete decompress sublocation.
-                    rimraf(internalConfig.tempFilesLocation, rimrafErr => consoleError(rimrafErr, internalConfig.outputErrorToConsole));
+                if (!internalConfig.preserveTempFiles) {
+                    /** Safely delete location */
+                    function safelyDeleteLocation(location, deleteDirIfEmpty) {
+                        if (!fs.existsSync(location))
+                            return;
+
+                        if (!fs.lstatSync(location).isDirectory()       // If not directory
+                            || !deleteDirIfEmpty                        // or if the deleteDirIfEmpty is false or undefined
+                            || fs.readdirSync(location).length == 0) {  // or if it is true, we check if the contents are empty.
+                            try {
+                                rimrafSync(location);
+                            }
+                            catch(rimrafErr) {
+                                consoleError(rimrafErr, internalConfig.outputErrorToConsole);
+                            }
+                        }
+                    }
+
+                    // We delete our file as well as the extracted files inside the office files.
+                    // There is no extraction for pdf files because we don't support any unpacking of it.
+                    // After removing files, we check if the folders containing them are empty.
+                    // If yes, we remove those folders too.
+                    safelyDeleteLocation(filepath);
+                    safelyDeleteLocation(getFilePath(internalConfig.tempFilesLocation, filename));
+                    safelyDeleteLocation(getTempFilesDirectory(internalConfig.tempFilesLocation), true);
+                    safelyDeleteLocation(internalConfig.tempFilesLocation, true);
+                }
 
                 // Check if there is an error. Throw if there is an error.
                 if (err)
@@ -597,21 +670,30 @@ function parseOfficeAsync(file, config = {}) {
 let globalFileNameIterator = 0;
 /**
  * File Name generator that takes the extension as an input and returns a file name that comprises a timestamp and an incrementing number
- * to allow the files to be sorted in chronological order
- * @param {string} tempFilesLocation Directory whether this new file needs to be stored
- * @param {string} ext               File extension for this new generated file name
+ * to allow the files to be sorted in chronological order. We also prefix them with ppid and pid of the process to support
+ * common destination from worker threads as well as multiple processes running them together.
+ * @param {string} ext File extension for this new generated file name
  * @returns {string}
  */
-function getNewFileName(tempFilesLocation, ext) {
+function getNewFileName(ext) {
     // Get the iterator part of the file name
     let iteratorPart = (globalFileNameIterator++).toString().padStart(5, '0');
     // We want the iterator part of the file name to be of 5 digits.
     // Therefore, when the iterator crosses into 6 digits, we reset it to 0.
     if (globalFileNameIterator > 99999)
         globalFileNameIterator = 0;
+    // Return the file name with ppid and pid to allow unique names even with worker threads.
+    return `${process.ppid}_${process.pid}_${new Date().getTime().toString() + iteratorPart}.${ext}`;
+}
+
+/** Gets directory for storing files. */
+function getTempFilesDirectory(root) {
+    return `${root}/tempfiles`;
+}
 
-    // Return the file name
-    return `${tempFilesLocation}/tempfiles/${new Date().getTime().toString() + iteratorPart}.${ext}`;
+/** Gets file path for the supplied directory and the file name. */
+function getFilePath(directory, fileName) {
+    return `${directory}/${fileName}`;
 }
 
 /**
@@ -634,7 +716,7 @@ module.exports.parseOfficeAsync = parseOfficeAsync;
 
 
 // Run this library on CLI
-if ((process.argv[0].split('/').pop() == "node" || process.argv[0].split('/').pop() == "npx") && (process.argv[1].split('/').pop() == "officeParser.js" || process.argv[1].split('/').pop() == "officeparser")) {
+if ((process.argv[0].split('/').pop() == "node" || process.argv[0].split('/').pop() == "npx") && (process.argv[1].split('/').pop() == "officeParser.js" || process.argv[1].split('/').pop().toLowerCase() == "officeparser")) {
     if (process.argv.length == 2) {
         // continue
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "officeparser",
-  "version": "4.1.1",
+  "version": "4.2.0",
   "description": "A Node.js library to parse text out of any office file. Currently supports docx, pptx, xlsx, odt, odp, ods, pdf files.",
   "main": "officeParser.js",
   "files": [
@@ -44,10 +44,10 @@
   "homepage": "https://github.com/harshankur/officeParser#readme",
   "dependencies": {
     "@xmldom/xmldom": "^0.8.10",
-    "decompress": "^4.2.0",
+    "decompress": "^4.2.1",
     "file-type": "^16.5.4",
     "node-ensure": "^0.0.0",
-    "rimraf": "^2.6.3"
+    "rimraf": "^5.0.10"
   },
   "devDependencies": {
     "@types/decompress": "^4.2.6",