@iebh/reflib 2.5.3 → 2.5.4

package/README.md

@@ -1,5 +1,5 @@
-Reflib
-======
+@IEBH/Reflib
+============
 Reference library processing for Node.
 
 This library provides various read/write functionality to process citation libraries and handle individual references (henceforth "Refs").
@@ -13,8 +13,8 @@ Compatibility
 | Library                | Extension(s)    | Read               | Write              |
 |------------------------|-----------------|--------------------|--------------------|
 | Comma Separated Values | `.csv`          | :x:                | :x:                |
-| EndNote ENL            | `.enl`          | :x:                | :x:                |
-| EndNote ENLX           | `.enlx`         | :x:                | :x:                |
+| EndNote ENL            | `.enl`          | :heavy_check_mark: | (untested)         |
+| EndNote ENLX           | `.enlx`         | :heavy_check_mark: | :x:                |
 | EndNote XML            | `.xml`          | :heavy_check_mark: | :heavy_check_mark: |
 | JSON                   | `.json`         | :heavy_check_mark: | :heavy_check_mark: |
 | Medline                | `.nbib`         | :heavy_check_mark: | :heavy_check_mark: |

package/lib/formats.js

@@ -18,6 +18,22 @@ export let formats = {
 		canRead: false,
 		canWrite: false,
 	},
+	endnoteEnl: {
+		id: 'endnoteEnl',
+		title: 'EndNote ENL',
+		titleShort: 'EndNote',
+		ext: ['.enl'],
+		canRead: true,
+		canWrite: true,
+	},
+	endnoteEnlX: {
+		id: 'endnoteEnlX',
+		title: 'EndNote ENLX',
+		titleShort: 'EndNote ENLX',
+		ext: ['.enlx'],
+		canRead: true,
+		canWrite: false,
+	},
 	endnoteXml: {
 		id: 'endnoteXml',
 		title: 'EndNoteXML',

package/lib/writeFile.js

@@ -1,4 +1,5 @@
 import {createWriteStream} from 'node:fs';
+import {finished as streamFinished} from 'node:stream/promises';
 import {identifyFormat} from './identifyFormat.js';
 import {writeStream} from './writeStream.js';
 
@@ -16,11 +17,15 @@ export function writeFile(path, refs, options) {
 	let format = options?.module || identifyFormat(path)?.id;
 	if (!format) throw new Error(`Unable to identify reference library format when saving file "${path}"`);
 
-	let writer = writeStream(format, createWriteStream(path), options);
+	let fileStream = createWriteStream(path);
+	let writer = writeStream(format, fileStream, options);
+
 	return Promise.resolve()
 		.then(()=> writer.start())
 		.then(()=> refs.reduce((chain, ref) => // Write all refs as a series of promises
 			chain.then(()=> writer.write(ref))
 		, Promise.resolve()))
 		.then(()=> writer.end())
+		.then(()=> fileStream.close())
+		.then(()=> streamFinished(fileStream))
 }

package/modules/default.js

@@ -1,4 +1,6 @@
 export * as json from './json.js';
+export * as endnoteEnl from './endnoteEnl.js';
+export * as endnoteEnlX from './endnoteEnlX.js';
 export * as endnoteXml from './endnoteXml.js';
 export * as medline from './medline.js';
 export * as ris from './ris.js';

package/modules/endnoteEnl.js

@@ -0,0 +1,237 @@
+import Emitter from '../shared/emitter.js';
+import {default as SQLite} from 'sql.js';
+
+/**
+* EndNote/SDB column to RefLib column mappings
+* This must match EXACTLY a known, working .sdb Sqlite database table pragma
+* Column ID's not present within this list are generally scrapped
+*
+* @type {Array<Object>}
+* @property {String} sl The Sqlite/SDB column within the `refs` table
+* @property {String} rl The Reflib column (see README.md)
+* @property {'TEXT'|'INTEGER'} [type='TEXT'] SQLite column type, used for encoding when writing
+* @property {Function} [value] Optional column value mapper. Called as `(val:Any, ref:Object, col:Object, colIndex:Number, refIndex:Number)`
+* @property {*} [slDefault=''] Overriding SQLite column value if the field is empty
+*/
+export const columnMappings = [
+	// FIXME: This list is incomplete
+	{sl: 'id', rl: 'recNumber', type: 'INTEGER', value: (val, ref, col, colIndex, refIndex) => ref.recNumber || refIndex + 1},
+	// 'trash_state',
+	// 'text_styles',
+	// 'reference_type',
+	{sl: 'author', rl: 'author'},
+	// 'year',
+	{sl: 'title', rl: 'title'},
+	{sl: 'pages', rl: 'pages'},
+	// 'secondary_title',
+	{sl: 'volume', rl: 'volume'},
+	{sl: 'number', rl: 'number'},
+	// 'number_of_volumes',
+	// 'secondary_author',
+	// 'place_published',
+	// 'publisher',
+	// 'subsidiary_author',
+	// 'edition',
+	// 'keywords',
+	// 'type_of_work',
+	{sl: 'date', rl: 'date'},
+	{sl: 'abstract', rl: 'abstract'},
+	{sl: 'label', rl: 'label'},
+	// 'url',
+	// 'tertiary_title',
+	// 'tertiary_author',
+	{sl: 'notes', rl: 'notes'},
+	{sl: 'isbn', rl: 'isbn'},
+	{sl: 'custom_1', rl: 'custom1'},
+	{sl: 'custom_2', rl: 'custom2'},
+	{sl: 'custom_3', rl: 'custom3'},
+	{sl: 'custom_4', rl: 'custom4'},
+	// 'alternate_title',
+	// 'accession_number',
+	// 'call_number',
+	// 'short_title',
+	{sl: 'custom_5', rl: 'custom5'},
+	{sl: 'custom_6', rl: 'custom6'},
+	{sl: 'section', rl: 'section'},
+	// 'original_publication',
+	// 'reprint_edition',
+	// 'reviewed_item',
+	// 'author_address',
+	{sl: 'caption', rl: 'caption'},
+	{sl: 'custom_7', rl: 'custom7'},
+	// 'electronic_resource_number',
+	// 'translated_author',
+	// 'translated_title',
+	// 'name_of_database',
+	// 'database_provider',
+	{sl: 'research_notes', rl: 'researchNotes'},
+	{sl: 'language', rl: 'language'},
+	// 'access_date',
+	// 'last_modified_date',
+	// 'record_properties',
+	// 'added_to_library',
+	// 'record_last_updated',
+	// 'reserved3',
+	// 'fulltext_downloads',
+	// 'read_status',
+	// 'rating',
+	// 'reserved7',
+	// 'reserved8',
+	// 'reserved9',
+	// 'reserved10'
+];
+
+export const baseSql = [
+	'CREATE TABLE refs(id INTEGER PRIMARY KEY AUTOINCREMENT,trash_state INTEGER NOT NULL DEFAULT 0,text_styles TEXT NOT NULL DEFAULT "",reference_type INTEGER NOT NULL DEFAULT 0,author TEXT NOT NULL DEFAULT "",year TEXT NOT NULL DEFAULT "",title TEXT NOT NULL DEFAULT "",pages TEXT NOT NULL DEFAULT "",secondary_title TEXT NOT NULL DEFAULT "",volume TEXT NOT NULL DEFAULT "",number TEXT NOT NULL DEFAULT "",number_of_volumes TEXT NOT NULL DEFAULT "",secondary_author TEXT NOT NULL DEFAULT "",place_published TEXT NOT NULL DEFAULT "",publisher TEXT NOT NULL DEFAULT "",subsidiary_author TEXT NOT NULL DEFAULT "",edition TEXT NOT NULL DEFAULT "",keywords TEXT NOT NULL DEFAULT "",type_of_work TEXT NOT NULL DEFAULT "",date TEXT NOT NULL DEFAULT "",abstract TEXT NOT NULL DEFAULT "",label TEXT NOT NULL DEFAULT "",url TEXT NOT NULL DEFAULT "",tertiary_title TEXT NOT NULL DEFAULT "",tertiary_author TEXT NOT NULL DEFAULT "",notes TEXT NOT NULL DEFAULT "",isbn TEXT NOT NULL DEFAULT "",custom_1 TEXT NOT NULL DEFAULT "",custom_2 TEXT NOT NULL DEFAULT "",custom_3 TEXT NOT NULL DEFAULT "",custom_4 TEXT NOT NULL DEFAULT "",alternate_title TEXT NOT NULL DEFAULT "",accession_number TEXT NOT NULL DEFAULT "",call_number TEXT NOT NULL DEFAULT "",short_title TEXT NOT NULL DEFAULT "",custom_5 TEXT NOT NULL DEFAULT "",custom_6 TEXT NOT NULL DEFAULT "",section TEXT NOT NULL DEFAULT "",original_publication TEXT NOT NULL DEFAULT "",reprint_edition TEXT NOT NULL DEFAULT "",reviewed_item TEXT NOT NULL DEFAULT "",author_address TEXT NOT NULL DEFAULT "",caption TEXT NOT NULL DEFAULT "",custom_7 TEXT NOT NULL DEFAULT "",electronic_resource_number TEXT NOT NULL DEFAULT "",translated_author TEXT NOT NULL DEFAULT "",translated_title TEXT NOT NULL DEFAULT "",name_of_database TEXT NOT NULL DEFAULT "",database_provider TEXT NOT NULL DEFAULT "",research_notes TEXT NOT NULL DEFAULT "",language TEXT NOT NULL DEFAULT "",access_date TEXT NOT NULL DEFAULT "",last_modified_date TEXT NOT NULL DEFAULT "",record_properties TEXT NOT NULL DEFAULT "",added_to_library INTEGER NOT NULL DEFAULT 0,record_last_updated INTEGER NOT NULL DEFAULT 0,reserved3 INTEGER NOT NULL DEFAULT 0,fulltext_downloads TEXT NOT NULL DEFAULT "",read_status TEXT NOT NULL DEFAULT "",rating TEXT NOT NULL DEFAULT "",reserved7 TEXT NOT NULL DEFAULT "",reserved8 TEXT NOT NULL DEFAULT "",reserved9 TEXT NOT NULL DEFAULT "",reserved10 TEXT NOT NULL DEFAULT "")',
+].join(';');
+
+/**
+* Lookup object for Sqlite columns to the column mapping object
+*
+* @type {Object<Object>} Each column mapping item with the Sqlite column name as the key
+*/
+export const columnMappingSL2RL = Object.fromEntries(
+	columnMappings
+		.map(cm => [cm.sl, cm])
+)
+
+
+/**
+* Read an EndNote(@11+) / SQLite database, returning an Emitter analogue
+*
+* @see modules/inhterface.js
+*
+* @param {Stream} stream Stream primative to encapsulate
+*
+* @returns {Object} An Emitter analogue defined in `../shared/Emitter.js`
+*/
+export function readStream(stream) {
+	let emitter = Emitter();
+
+	// Queue up the parser in the next tick (so we can return the emitter first)
+	setTimeout(()=> {
+		let chunks = []; // Gathered buffer chunks, used to make an arrayBuffer later
+
+		stream
+			.on('data', chunk => chunks.push(chunk))
+			.on('error', e => emitter.emit('error', e))
+			.on('end', ()=> {
+				Promise.resolve()
+					.then(()=> { // Parse chunks into an arrayBuffer
+						let buf = new Uint8Array(
+							chunks.reduce((total, chunk) => total + chunk.length, 0)
+						);
+
+						let position = 0;
+						chunks.forEach(chunk => {
+							buf.set(chunk, position);
+							position += chunk.length;
+						});
+
+						// Release chunks to free up memory
+						chunks = [];
+
+						return buf;
+					})
+					.then(buf => this.readBuffer(buf, { // Hand arrayBuffer off to readBuffer() for processing
+						onRef(ref) {
+							emitter.emit('ref', ref);
+						},
+					}))
+					.finally(()=> emitter.emit('end'))
+			})
+	});
+
+	return emitter;
+}
+
+
+/**
+* Actual reader function
+* Accepts a buffer and tiggers the emitter when references are extracted + when ending
+*
+* @param {Uint8Array} buf The buffer to process
+*
+* @param {Object} [options] Additional options to mutate behaviour
+* @param {Function} [options.onRef] Function to trigger when we extract a reference from the buffer. Called as `(ref:RefLibRef)`
+* @param {String} [options.refTable='enl_refs'] The name of the reference table to query
+*
+* @returns {Promise} A promise which will eventually resolve when the read operation on the buffer has completed
+*/
+export function readBuffer(buf, options) {
+	let settings = {
+		refTable: 'refs',
+		onRef() {},
+		...options,
+	};
+	return Promise.resolve()
+		.then(()=> SQLite()) // Init database
+		.then(sqli => new sqli.Database(buf)) // Create SQLite database
+		.then(db => db.exec(`SELECT * from ${settings.refTable}`)) // Slurp all references
+		.then(([{columns, values}]) => { // Digest references
+			values.forEach(v => {
+				let ref = columns.reduce((ref, col, colIndex) => {
+					if (columnMappingSL2RL[col]) // Is a column we have a mapping for (implied else - ignore the data)
+						ref[columnMappingSL2RL[col].rl] = v[colIndex];
+
+					return ref;
+				}, {});
+
+				settings.onRef(ref);
+			})
+		})
+}
+
+
+/**
+* Write references to an EndNote(@11+)) SQLite database file
+*
+* @see modules/interface.js
+*
+* @param {Stream} stream Writable stream to output to
+*
+* @returns {Object} A writable stream analogue defined in `modules/interface.js`
+*/
+export function writeStream(stream) {
+	let db; // Database we are writing to
+	let refIndex = 0;
+	let insertOp; // Prepared query to insert a single ref
+
+	return {
+		start() {
+			return SQLite()
+				.then(sqli => new sqli.Database())
+				.then(res => db = res)
+				.then(()=> db.exec(baseSql))
+				.then(()=> insertOp = db.prepare(
+					'INSERT INTO refs'
+					+ '('
+						+ columnMappings.map(cm => cm.sl).join(', ')
+					+ ') '
+					+ 'VALUES ('
+						+ columnMappings.map(cm => cm.sl).map(k => ':' + k).join(', ')
+					+ ')'
+				))
+		},
+
+		write(ref) {
+			// Compose ref object we are going to throw at SQLite
+			let slRef = columnMappings
+				.reduce((r, col, colIndex) => {
+					r[':' + col.sl] =
+						col.value ? col.value(ref[col.rl], ref, col, colIndex, refIndex++)
+						: ref[col.rl] || col.slDefault || '';
+
+					return r;
+				}, {});
+
+			return insertOp.run(slRef);
+		},
+
+		end() {
+			return Promise.resolve()
+				.then(()=> stream.write(db.export()))
+				.then(()=> stream.end());
+		},
+	};
+}

package/modules/endnoteEnlX.js

@@ -0,0 +1,85 @@
+import * as EndNoteEnl from './endnoteEnl.js';
+import Emitter from '../shared/emitter.js';
+import {BlobReader as ZipBlobReader, ZipReader, Writer} from '@zip.js/zip.js';
+
+/**
+* Utility class to read a Zip file into a Uint8Array
+* This extends the deafult @zip.js/zip.js Writer class so its works without a crowbar
+*/
+class BinaryStringWriter extends Writer {
+	chunks = [];
+
+	writeUint8Array(chunk) {
+		this.chunks.push(chunk);
+	}
+
+	getData() {
+		let buf = new Uint8Array(
+			this.chunks.reduce((total, chunk) => total + chunk.length, 0)
+		);
+
+		let position = 0;
+		this.chunks.forEach(chunk => {
+			buf.set(chunk, position);
+			position += chunk.length;
+		});
+
+		// Flush buffer to free up memory
+		this.chunks = [];
+
+		return buf;
+	}
+}
+
+
+/**
+* Read an EndNote(@11+) / Zip+SQLite database EnlX returning an Emitter analogue
+*
+* Since EnlX files are really just a Zip file containing the SQLite database we're actually intrestesd in, most of this module
+* is handling the Zip container and handing the extracted buffer to the regular modules/endnoteEnl.js driver
+*
+* @see modules/inhterface.js
+*
+* @param {Stream} stream Stream primative to encapsulate
+*
+* @returns {Object} An Emitter analogue defined in `../shared/Emitter.js`
+*/
+export function readStream(stream) {
+	let emitter = Emitter();
+
+	// Queue up the parser in the next tick (so we can return the emitter first)
+	setTimeout(()=> {
+		let chunks = []; // Gathered buffer chunks, used to make an arrayBuffer later
+
+		stream
+			.on('data', chunk => chunks.push(chunk))
+			.on('error', e => emitter.emit('error', e))
+			.on('end', ()=> {
+				Promise.resolve()
+					.then(()=> { // Parse chunks into a Blob
+						let blob = new Blob(chunks);
+
+						// Release chunks to free up memory
+						chunks = [];
+						return blob;
+					})
+					.then(blob => new ZipReader( // Create zipReader
+						new ZipBlobReader(blob),
+					))
+					.then(zip => zip.getEntries()) // Fetch files
+					.then(files => files.find(f => f.filename == 'sdb/sdb.eni')) // Find the file we want
+					.then(sdb => { // Extract that files stream
+						let writer = new BinaryStringWriter();
+						return sdb.getData(writer);
+					})
+					.then(buf => EndNoteEnl.readBuffer(buf, {
+						onRef(ref) {
+							emitter.emit('ref', ref);
+						},
+					}))
+					.finally(()=> emitter.emit('end'))
+			})
+	});
+
+	return emitter;
+}

package/modules/endnoteXml.js

@@ -5,10 +5,11 @@ import Emitter from '../shared/emitter.js';
 import { WritableStream as XMLParser } from 'htmlparser2/lib/WritableStream';
 
 /**
-* Read an EndnoteXML file, returning a stream analogue
+* Read an EndnoteXML file, returning an Emitter analogue
+*
 * @see modules/inhterface.js
 * @param {Stream} stream Stream primative to encapsulate
-* @returns {Object} A readable stream analogue defined in `modules/interface.js`
+* @returns {Object} An Emitter analogue defined in `../shared/Emitter.js`
 */
 export function readStream(stream) {
 	let emitter = Emitter();
@@ -121,6 +122,8 @@ export function readStream(stream) {
 
 
 /**
+* Write references to a file
+*
 * @see modules/interface.js
 *
 * @param {Stream} stream Writable stream to output to

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@iebh/reflib",
-  "version": "2.5.3",
+  "version": "2.5.4",
   "description": "Reference / Citation reference library utilities",
   "scripts": {
     "lint": "eslint .",
     "test": "mocha",
-    "test:browser": "cd test/browser && npm run dev"
+    "test:browser": "cd test/browser && npm run dev",
+    "test:watch": "nodemon --exec npm run test"
   },
   "repository": {
     "type": "git",
@@ -44,18 +45,21 @@
     "JSONStream": "./modules/shims/JSONStream-browser.js"
   },
   "devDependencies": {
-    "@momsfriendlydevco/eslint-config": "^2.0.3",
-    "chai": "^5.1.1",
-    "eslint": "^9.9.0",
-    "mocha": "^10.7.3",
+    "@momsfriendlydevco/eslint-config": "^2.1.2",
+    "chai": "^5.2.0",
+    "eslint": "^9.21.0",
+    "mocha": "^11.1.0",
     "mocha-logger": "^1.0.8",
+    "nodemon": "^3.1.9",
     "temp": "^0.9.4",
     "vite-plugin-replace": "^0.1.1"
   },
   "dependencies": {
-    "@iebh/cacx": "^1.0.2",
+    "@iebh/cacx": "^1.0.3",
+    "@zip.js/zip.js": "^2.7.57",
     "htmlparser2": "^9.1.0",
     "JSONStream": "^1.3.5",
-    "mitt": "^3.0.1"
+    "mitt": "^3.0.1",
+    "sql.js": "^1.12.0"
   }
 }