@iebh/reflib 2.0.4 → 2.1.0

package/README.md

@@ -193,6 +193,19 @@ reflib.readFile('./data/json/json1.json')
 	.then(refs => /* Do something with Ref collection */)
 ```
 
+An emitter is available to track progress while reading. Note that due to the chainable nature of promises the first return contains the `emitter` key only:
+
+```javascript
+let reader = reflib.readFile('./data/json/json1.json');
+
+reader.emitter
+	.on('progress', ({readBytes, totalSize, refsFound}) => /* Report progress somehow */);
+	.on('end', ({refsFound}) => /* Report progress somehow */);
+
+reader
+	.then(refs => /* Do something with Ref collection */)
+```
+
 uploadFile(options)
 ===================
 Available: Browser
@@ -257,12 +270,3 @@ Credits
 =======
 Developed for the [Bond University Institute for Evidence-Based Healthcare](https://iebh.bond.edu.au).
 Please contact [the author](mailto:matt_carter@bond.edu.au) with any issues.
-
-
-TODO
-====
-- [x] Basic parsing iterfaces
-- [ ] "progress" emitter for files
-- [x] `.uploadFile()` browser compatibility
-- [ ] `.downloadFile()` browser compatibility
-- [x] `setup()` functions per module to avoid things like map calculations unless the module is actually needed

package/lib/readFile.js

@@ -1,4 +1,5 @@
 import {createReadStream} from 'node:fs';
+import Emitter from '../shared/emitter.js';
 import {stat} from 'node:fs/promises';
 import {identifyFormat} from './identifyFormat.js';
 import {readStream} from './readStream.js';
@@ -9,16 +10,26 @@ import {readStream} from './readStream.js';
 * @param {string} path The file path to parse
 * @param {Object} [options] Additional options to pass to the parser
 * @param {string} [options.module] The module to use if overriding from the file path
+*
 * @returns {Promise<Array>} An eventual array of all references parsed from the file
+* @property {EventEmitter} emitter An event emitter which will fire the below events
+*
+* @fires progress Emitted as `({readBytes: Number, totalSize: Number, refsFound: Number})`
+* @fires end Emitted as `({refsFound: Number})` when the reading operation has completed
 */
 export function readFile(path, options) {
-	let settings = {...options};
+	let settings = {
+		progressTotal: false,
+		...options,
+	};
 	let module = options?.module || identifyFormat(path)?.id;
 	if (!module) throw new Error(`Unable to identify reference library format for file path "${path}"`);
 
-	return stat(path)
+	let promiseEmitter = Promise.resolve()
+		.then(()=> stat(path))
 		.then(stats => new Promise((resolve, reject) => {
-			let refs = [];
+			let refs = []; // eslint-disable-line no-unused-vars
+
 			readStream(
 				module,
 				createReadStream(path),
@@ -30,5 +41,23 @@ export function readFile(path, options) {
 				.on('end', ()=> resolve(refs))
 				.on('error', reject)
 				.on('ref', ref => refs.push(ref))
+				.on('progress', readBytes => promiseEmitter.emitter.emit('progress', {
+					readBytes,
+					totalSize: stats.size,
+					refsFound: refs.length,
+				}))
 		}))
+		.then(refs => {
+			promiseEmitter.emitter.emit('end', {refsFound: refs.length})
+			return refs;
+		})
+
+	// Extend our base promise with an emitter subkey
+	return Object.defineProperties(promiseEmitter, {
+		emitter: {
+			value: Emitter(),
+			enumerable: true,
+			writable: false,
+		},
+	});
 }

package/lib/uploadFile.js

@@ -6,7 +6,7 @@ import StreamEmitter from '../shared/streamEmitter.js';
 /**
 * Prompt the user for a file then read it as a Reflib event emitter
 * @param {Object} [options] Additional options when prompting the user
-* @param {File} [options.files] The File object to process, omitting this will prompt the user to select a file
+* @param {File} [options.file] The File object to process, omitting this will prompt the user to select a file
 * @param {function} [options.onStart] Async function called as `(File)` when starting the read stage
 * @param {function} [options.onProgress] Function called as `(position, totalSize)` when processing the file
 * @param {function} [options.onEnd] Async function called as `()` when the read stage has completed
@@ -49,9 +49,10 @@ export function uploadFile(options) {
 		return Promise.resolve()
 			.then(()=> settings.onStart && settings.onStart(settings.file))
 			.then(()=> new Promise((resolve, reject) => {
+
 				let streamer = readStream(
 					identifiedType.id,
-					StreamEmitter(settings.file.stream().getReader()),
+					StreamEmitter(settings.file.stream()),
 					{
 						...settings,
 						size: settings.file.size,

package/modules/endnoteXml.js

@@ -1,7 +1,9 @@
 import camelCase from '../shared/camelCase.js';
 import Emitter from '../shared/emitter.js';
-import {WritableStream as XMLParser} from 'htmlparser2/lib/WritableStream';
 
+// TODO: CF: Don't need to import both, it depends if we are on browser or node
+import * as htmlparser2 from "htmlparser2";
+import {WritableStream as XMLParser} from 'htmlparser2/lib/WritableStream';
 
 /**
 * @see modules/interface.js
@@ -30,75 +32,112 @@ export function readStream(stream) {
 	*/
 	let textAppend = false;
 
+	/**
+	 * The options/callbacks for the parser
+	 * @type {Object}
+	 */
+	let parserOptions = {
+		xmlMode: true,
+		decodeEntities: false, // Handled below
+		onopentag(name, attrs) {
+			textAppend = false;
+			stack.push({
+				name: camelCase(name),
+				attrs,
+			});
+		},
+		onclosetag(name) {
+			if (name == 'record') {
+				if (ref.title) ref.title = ref.title // htmlparser2 handles the '<title>' tag in a really bizare way so we have to pull apart the <style> bits when parsing
+					.replace(/^.*<style.*>(.*)<\/style>.*$/m, '$1')
+					.replace(/^\s+/, '')
+					.replace(/\s+$/, '')
+				emitter.emit('ref', translateRawToRef(ref));
+				stack = []; // Trash entire stack when hitting end of <record/> node
+				ref = {}; // Reset the ref state
+			} else {
+				stack.pop();
+			}
+		},
+		ontext(text) {
+			let parentName = stack[stack.length - 1]?.name;
+			let gParentName = stack[stack.length - 2]?.name;
+			if (parentName == 'title') {
+				if (textAppend) {
+					ref.title += text;
+				} else {
+					ref.title = text;
+				}
+			} else if (parentName == 'style' && gParentName == 'author') {
+				if (!ref.authors) ref.authors = [];
+				if (textAppend) {
+					ref.authors[ref.authors.length - 1] += xmlUnescape(text);
+				} else {
+					ref.authors.push(xmlUnescape(text));
+				}
+			} else if (parentName == 'style' && gParentName == 'keyword') {
+				if (!ref.keywords) ref.keywords = [];
+				if (textAppend) {
+					ref.keywords[ref.keywords.length - 1] += xmlUnescape(text);
+				} else {
+					ref.keywords.push(xmlUnescape(text));
+				}
+			} else if (parentName == 'style') { // Text within <style/> tag
+				if (textAppend || ref[gParentName]) { // Text already exists? Append (handles node-expats silly multi-text per escape character "feature")
+					ref[gParentName] += xmlUnescape(text);
+				} else {
+					ref[gParentName] = xmlUnescape(text);
+				}
+			} else if (['recNumber', 'refType'].includes(parentName)) { // Simple setters like <rec-number/>
+				if (textAppend || ref[parentName]) {
+					ref[parentName] += xmlUnescape(text);
+				} else {
+					ref[parentName] = xmlUnescape(text);
+				}
+			}
+			textAppend = true; // Always set the next call to the text emitter handler as an append operation
+		},
+		onend() {
+			emitter.emit('end');
+		}
+	}
 
 	// Queue up the parser in the next tick (so we can return the emitter first)
-	setTimeout(()=> {
-		let parser = new XMLParser({
-			xmlMode: true,
-			decodeEntities: false, // Handled below
-			onopentag(name, attrs) {
-				textAppend = false;
-				stack.push({
-					name: camelCase(name),
-					attrs,
-				});
-			},
-			onclosetag(name) {
-				if (name == 'record') {
-					if (ref.title) ref.title = ref.title // htmlparser2 handles the '<title>' tag in a really bizare way so we have to pull apart the <style> bits when parsing
-						.replace(/^.*<style.*>(.*)<\/style>.*$/m, '$1')
-						.replace(/^\s+/, '')
-						.replace(/\s+$/, '')
-					emitter.emit('ref', translateRawToRef(ref));
-					stack = []; // Trash entire stack when hitting end of <record/> node
-					ref = {}; // Reset the ref state
+	setTimeout(() => {
+
+		if (typeof stream.pipe === 'function') {
+			// We are on the node.js client
+			let parser = new XMLParser(parserOptions);
+			stream.on('data', ()=> emitter.emit('progress', stream.bytesRead))
+			stream.pipe(parser)
+			return;
+		}
+
+		// TODO: CF: We may want to consider moving to a DIY parser for speed and memory efficiency
+		if (typeof stream.getReader === 'function') {
+			// We are on the browser
+			var reader = stream.getReader();
+			var parser = new htmlparser2.Parser(parserOptions);
+			parseXMLOnBrowser();
+			return;
+		}
+
+		function parseXMLOnBrowser() {
+			reader.read().then(({done, value}) => {
+				if (done) {
+					parser.end();
 				} else {
-					stack.pop();
-				}
-			},
-			ontext(text) {
-				let parentName = stack[stack.length - 1]?.name;
-				let gParentName = stack[stack.length - 2]?.name;
-				if (parentName == 'title') {
-					if (textAppend) {
-						ref.title += text;
-					} else {
-						ref.title = text;
-					}
-				} else if (parentName == 'style' && gParentName == 'author') {
-					if (!ref.authors) ref.authors = [];
-					if (textAppend) {
-						ref.authors[ref.authors.length - 1] += xmlUnescape(text);
-					} else {
-						ref.authors.push(xmlUnescape(text));
-					}
-				} else if (parentName == 'style' && gParentName == 'keyword') {
-					if (!ref.keywords) ref.keywords = [];
-					if (textAppend) {
-						ref.keywords[ref.keywords.length - 1] += xmlUnescape(text);
-					} else {
-						ref.keywords.push(xmlUnescape(text));
-					}
-				} else if (parentName == 'style') { // Text within <style/> tag
-					if (textAppend || ref[gParentName]) { // Text already exists? Append (handles node-expats silly multi-text per escape character "feature")
-						ref[gParentName] += xmlUnescape(text);
-					} else {
-						ref[gParentName] = xmlUnescape(text);
-					}
-				} else if (['recNumber', 'refType'].includes(parentName)) { // Simple setters like <rec-number/>
-					if (textAppend || ref[parentName]) {
-						ref[parentName] += xmlUnescape(text);
-					} else {
-						ref[parentName] = xmlUnescape(text);
-					}
+					var text = new TextDecoder().decode(value);
+					parser.write(text);
+					text = null; // Free up memory
+					parseXMLOnBrowser();
 				}
-				textAppend = true; // Always set the next call to the text emitter handler as an append operation
-			},
-		})
+			})
+		}
 
-		stream.pipe(parser)
-			.on('finish', ()=> emitter.emit('end'))
-	});
+		console.error("This line should not be hit!");
+
+	})
 
 	return emitter;
 }

package/modules/interface.js

@@ -14,6 +14,7 @@
 * @emits ref Emitted with a single ref object when found
 * @emits end Emitted when parsing has completed
 * @emits error Emitted when an error has been raised
+* @emits progress Emitted as (bytesRead) when reading a stream
 */
 export function readStream(stream, options) {
 	// Stub

package/modules/json.js

@@ -9,7 +9,8 @@ export function readStream(stream) {
 	let emitter = Emitter();
 
 	// Queue up the parser in the next tick (so we can return the emitter first)
-	setTimeout(()=>
+	setTimeout(()=> {
+		stream.on('data', ()=> emitter.emit('progress', stream.bytesRead));
 		stream.pipe(
 			JSONStream.parse('*')
 				.on('data', ref => emitter.emit('ref', {
@@ -19,7 +20,7 @@ export function readStream(stream) {
 				.on('end', ()=> emitter.emit('end'))
 				.on('error', emitter.emit.bind('error'))
 		)
-	);
+	});
 
 	return emitter;
 }

package/modules/medline.js

@@ -122,6 +122,7 @@ export function readStream(stream, options) {
 	setTimeout(()=> {
 		stream
 			.on('data', chunkBuffer => {
+				emitter.emit('progress', stream.bytesRead);
 				buffer += chunkBuffer.toString(); // Append incomming data to the partial-buffer we're holding in memory
 
 				let bufferCrop = 0; // How many bytes to shift off the front of the buffer based on the last full reference we saw, should end up at the last byte offset of buffer that is valid to shift-truncate to

package/modules/ris.js

@@ -21,6 +21,7 @@ export function readStream(stream, options) {
 	setTimeout(()=> {
 		stream
 			.on('data', chunkBuffer => {
+				emitter.emit('progress', stream.bytesRead);
 				buffer += chunkBuffer.toString(); // Append incomming data to the partial-buffer we're holding in memory
 
 				let bufferCrop = 0; // How many bytes to shift off the front of the buffer based on the last full reference we saw, should end up at the last byte offset of buffer that is valid to shift-truncate to

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iebh/reflib",
-  "version": "2.0.4",
+  "version": "2.1.0",
   "description": "Reference / Citation reference library utilities",
   "scripts": {
     "lint": "eslint lib modules shared test",

package/shared/streamEmitter.js

@@ -3,7 +3,7 @@ import Emitter from '../shared/emitter.js';
 /**
 * Wrapper for streams which transforms a given input into an emitter pattern
 * This is designed to let regular `node:stream.Readable` objects pass through without alteration but browser based stream objects get wrapped
-* @param {stream.Readable|ReadableStreamDefaultReader} inStream The input stream to wrap
+* @param {stream.Readable|ReadableStream} inStream The input stream to wrap
 * @returns {stream.Readable|Emitter} Either the unedited node compatible stream or an event emitter with the same behaviour
 *
 * @emits data Emitted as `(chunk)` on each data chunk
@@ -11,27 +11,8 @@ import Emitter from '../shared/emitter.js';
 * @emits error Emitted as `(Error)` on any read error
 */
 export default function streamEmitter(inStream) {
-	if (inStream.on) return inStream; // inStream already supports event emitters - do nothing
-
-	let emitter = Emitter();
-	let utf8Decoder = new TextDecoder('utf-8');
-	let readCycle = ()=> {
-		inStream
-			.read()
-			.then(({value, done}) => {
-				if (done) {
-					emitter.emit('end');
-				} else {
-					emitter.emit('data', utf8Decoder.decode(value, {stream: true}));
-					setTimeout(readCycle); // Loop into next read if not already finished
-				}
-			})
-			.catch(e => emitter.emit('error', e))
-	};
-
-	// Keep downstream libraries happy by stubbing stream-like functions
-	emitter.setEncoding = ()=> {};
-
-	setTimeout(readCycle); // Queue up initial read cycle on next tick
-	return emitter;
+	// FIXME: Need to examine inStream and multiplex
+	//        inStream.pipeTo - a browser stream - passthru
+	//        !inStream.pipeTo - probably Node stream - need to glue pipeTo as a promiseable
+	return inStream;
 }