@iebh/reflib 2.0.0

package/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+    "env": {
+        "browser": true,
+        "es2021": true,
+		"mocha": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 13,
+        "sourceType": "module"
+    },
+    "rules": {
+    }
+};

package/.ignore

@@ -0,0 +1 @@
+test/data/

package/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Institute for Evidence-Based Healthcare
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,268 @@
+RefLib
+======
+Reference library processing for Node.
+
+This library provides various read/write functionality to process citation libraries and handle individual references (henceforth "Refs").
+
+**NOTE: THIS LIBRARY IS STILL UNDER CONSTRUCTION**
+**Please use [Version 1](https://github.com/hash-bang/Reflib-Node) ([NPM](https://www.npmjs.com/package/reflib)) until this message is removed**
+
+
+Compatibility
+=============
+
+| Library                | Extension(s)    | Read               | Write              |
+|------------------------|-----------------|--------------------|--------------------|
+| Comma Separated Values | `.csv`          | :x:                | :x:                |
+| EndNote ENL            | `.enl`          | :x:                | :x:                |
+| EndNote ENLX           | `.enlx`         | :x:                | :x:                |
+| EndNote XML            | `.xml`          | :heavy_check_mark: | :heavy_check_mark: |
+| JSON                   | `.json`         | :heavy_check_mark: | :heavy_check_mark: |
+| Medline                | `.nbib`         | :x:                | :x:                |
+| RIS                    | `.ris` / `.txt` | :heavy_check_mark: | :heavy_check_mark: |
+| Tab Separated Values   | `.tsv`          | :x:                | :x:                |
+
+
+**Notes on different formats**:
+
+* Medline seems to implement a totally different [publication type system](https://www.nlm.nih.gov/mesh/pubtypes.html) than others. RefLib will attempt to guess the best match, storing the original type in the `medlineType` key. Should the citation library be exported _back_ to Medline / `.nbib` files this key will take precedence to avoid data loss
+
+
+Reference Structure
+===================
+RefLib creates a simple Plain-Old-JavaScript-Object (POJO) for each reference it parses, or writes to a file format when given a collection of the same.
+
+Each reference has the following standardized fields, these are translated from whatever internal format each module uses - e.g. the `TY` RIS field is automatically translated to `title`.
+
+
+| Field            | Type            | Description                                                                            |
+|------------------|-----------------|----------------------------------------------------------------------------------------|
+| recNumber        | `number`        | The sorting number of the reference. Not present in RIS files                          |
+| type             | `string`        | A supported [reference type](#reference-types) (e.g. journalArticle)                   |
+| title            | `string`        | The reference's main title                                                             |
+| journal          | `string`        | The reference's secondary title, this is usually the journal for most published papers |
+| authors          | `array<string>` | An array of each Author in the originally specified format                             |
+| date             | `string`        | The raw, internal date of the reference                                                |
+| urls             | `array<string>` | An array of each URL for the reference                                                 |
+| pages            | `string`        | The page reference, usually in the format `123-4`                                      |
+| volume           | `string`        |                                                                                        |
+| number           | `string`        |                                                                                        |
+| isbn             | `string`        |                                                                                        |
+| abstract         | `string`        |                                                                                        |
+| label            | `string`        |                                                                                        |
+| caption          | `string`        |                                                                                        |
+| notes            | `string`        |                                                                                        |
+| address          | `string`        |                                                                                        |
+| researchNotes    | `string`        |                                                                                        |
+| keywords         | `array<string>` | Optional list of keywords that apply to the reference                                  |
+| accessDate       | `string`        |                                                                                        |
+| accession        | `string`        |                                                                                        |
+| doi              | `string`        |                                                                                        |
+| section          | `string`        |                                                                                        |
+| language         | `string`        |                                                                                        |
+| researchNotes    | `string`        |                                                                                        |
+| databaseProvider | `string`        |                                                                                        |
+| database         | `string`        |                                                                                        |
+| workType         | `string`        |                                                                                        |
+| custom1          | `string`        |                                                                                        |
+| custom2          | `string`        |                                                                                        |
+| custom3          | `string`        |                                                                                        |
+| custom4          | `string`        |                                                                                        |
+| custom5          | `string`        |                                                                                        |
+| custom6          | `string`        |                                                                                        |
+| custom7          | `string`        |                                                                                        |
+
+
+Reference Types
+---------------
+As with refs the following ref types are supported and translated from the module internal formats.
+
+
+```
+aggregatedDatabase
+ancientText
+artwork
+audioVisualMaterial
+bill
+blog
+book
+bookSection
+case
+catalog
+chartOrTable
+classicalWork
+computerProgram
+conferencePaper
+conferenceProceedings
+dataset
+dictionary
+editedBook
+electronicArticle
+electronicBook
+electronicBookSection
+encyclopedia
+equation
+figure
+filmOrBroadcast
+generic
+governmentDocument
+grant
+hearing
+journalArticle
+legalRuleOrRegulation
+magazineArticle
+manuscript
+map
+music
+newspaperArticle
+onlineDatabase
+onlineMultimedia
+pamphlet
+patent
+personalCommunication
+report
+serial
+standard
+statute
+thesis
+unknown
+unpublished
+web
+```
+
+
+
+API
+===
+Each API is available either from the default `reflib` object or as a separate import.
+
+
+```javascript
+import reflib from 'reflib'; // Import everything as `reflib`
+reflib.readFile(path);
+reflib.writeFile(path, refs);
+
+
+import {readFile, writeFile} from 'reflib'; // Import specific functions
+readFile(path);
+writeFile(path, refs);
+```
+
+
+formats
+=======
+Available: Node + Browser
+A lookup object of all supported citation library formats.
+Each key is the unique ID of that module.
+
+Properties are:
+
+| Key          | Type            | Description                                                            |
+|--------------|-----------------|------------------------------------------------------------------------|
+| `id`         | `String`        | The unique ID of that module (same as object key)                      |
+| `title`      | `String`        | Longer, human readable title of the module                             |
+| `titleShort` | `String`        | Shorter, human readable title of the module                            |
+| `ext`        | `Array<String>` | Array of output file extensions, first extension should be the default |
+| `canRead`    | `boolean`       | Whether the format is supported when reading a citation library        |
+| `canWrite`   | `boolean`       | Whether the format is supported when writing a citation library        |
+
+
+identifyFormat(path)
+====================
+Available: Node + Browser
+Attempt to determine the format of a file on disk from its path. The file does not need to actually exist.
+
+```javascript
+identifyFormat('My Refs.csv') //= csv
+identifyFormat('My Refs.json') //= json
+identifyFormat('My Refs.nbib') //= nbib
+identifyFormat('My Refs.txt.ris') //= ris
+identifyFormat('MY REFS.TXT.RIS') //= ris
+identifyFormat('My Refs.data.tsv') //= tsv
+identifyFormat('My Refs.xml') //= endnoteXml
+```
+
+
+readFile(path, options)
+=======================
+Available: Node
+Read a file on disk, returning a Promise which will resolve with an array of all Refs extracted.
+
+```javascript
+reflib.readFile('./data/json/json1.json')
+	.then(refs => /* Do something with Ref collection */)
+```
+
+uploadFile(options)
+===================
+Available: Browser
+Prompt the user for a file and process it into an array of citations.
+
+```javascript
+reflib.uploadFile({ // Additional options
+	file, // Optional File object if known, if omitted this function will prompt the user to select a file
+	onStart, // Async function called as `(File)` when starting the read stage
+	onProgress, // Function called as `(position, totalSize)` when processing the file
+	onEnd, // Async function called as `()` when the read stage has completed
+})
+	.then(refs => /* Do something with Ref collection */)
+```
+
+
+writeFile(path, refs, options)
+==============================
+Available: Node
+Write a file back to disk, returning a Promise which will resolve when done.
+
+```javascript
+reflib.writeFile('MyRefs.xml', refs);
+```
+
+
+
+readStream(moduleId, inputStream, options)
+==========================================
+Available: Node + Browser
+Low level worker of `readFile()`.
+Accept an input Stream.Readable and return a emitter which will emit each Ref found.
+
+```javascript
+reflib.readStream('json', createReadStream('./data/json/json1.json'))
+	.on('end', ()=> /* Finished reading */)
+	.on('error', err => /* Deal with errors */)
+	.on('ref', ref => /* Do something with extracted Ref */)
+```
+
+
+writeStream(moduleId, outputStream, options)
+============================================
+Available: Node + Browser
+Low level worker of `writeFile()`.
+Return an object with methods to call to write to a given stream.
+The returned object will have a `start()`, `end()` and `write(ref)` function which can be called to write to the original input stream.
+
+```javascript
+// Convert a JSON file to EndNoteXML via a stream
+let output = reflib.writeStream('json', createWriteStream('./MyRefs.xml'));
+
+output.start(); // Begin stream writing
+
+reflib.readStream('json', createReadStream('./data/json/json1.json'))
+	.on('ref', ref => output.write(ref))
+	.on('end', ()=> output.end())
+```
+
+
+Credits
+=======
+Developed for the [Bond University Institute for Evidence-Based Healthcare](https://iebh.bond.edu.au).
+Please contact [the author](mailto:matt_carter@bond.edu.au) with any issues.
+
+
+TODO
+====
+- [x] Basic parsing iterfaces
+- [ ] "progress" emitter for files
+- [x] `.uploadFile()` browser compatibility
+- [ ] `.downloadFile()` browser compatibility
+- [x] `setup()` functions per module to avoid things like map calculations unless the module is actually needed

package/lib/browser.js

@@ -0,0 +1,8 @@
+import {identifyFormat} from './identifyFormat.js';
+import {formats} from './formats.js';
+import {getModule} from './getModule.js';
+import {readStream} from './readStream.js';
+import {uploadFile} from './uploadFile.js';
+import {writeStream} from './writeStream.js';
+
+export default {identifyFormat, formats, getModule, readStream, uploadFile, writeStream};

package/lib/default.js

@@ -0,0 +1,7 @@
+export * from './identifyFormat.js';
+export * from './formats.js';
+export * from './getModule.js';
+export * from './readFile.js';
+export * from './readStream.js';
+export * from './writeFile.js';
+export * from './writeStream.js';

package/lib/fields.js

@@ -0,0 +1,161 @@
+/**
+* Field definitions for RefLib citations
+* @type {Object} An object lookup where each key represents a field within a citation
+* @property {string} type A TypeScript compatible type for that field
+* @property {array<string>} [value] Possible values if the type is restricted
+*/
+export let fields = {
+	recNumber: {
+		type: 'string',
+	},
+	type: {
+		type: 'string',
+		values: [
+			'aggregatedDatabase',
+			'ancientText',
+			'artwork',
+			'audioVisualMaterial',
+			'bill',
+			'blog',
+			'book',
+			'bookSection',
+			'case',
+			'catalog',
+			'chartOrTable',
+			'classicalWork',
+			'computerProgram',
+			'conferencePaper',
+			'conferenceProceedings',
+			'dataset',
+			'dictionary',
+			'editedBook',
+			'electronicArticle',
+			'electronicBook',
+			'electronicBookSection',
+			'encyclopedia',
+			'equation',
+			'figure',
+			'filmOrBroadcast',
+			'generic',
+			'governmentDocument',
+			'grant',
+			'hearing',
+			'journalArticle',
+			'legalRuleOrRegulation',
+			'agazineArticle',
+			'manuscript',
+			'map',
+			'music',
+			'newspaperArticle',
+			'onlineDatabase',
+			'onlineMultimedia',
+			'pamphlet',
+			'patent',
+			'personalCommunication',
+			'report',
+			'serial',
+			'standard',
+			'statute',
+			'thesis',
+			'unknown',
+			'unpublished',
+			'web',
+		],
+	},
+	title: {
+		type: 'string',
+	},
+	journal: {
+		type: 'string',
+	},
+	authors: {
+		type: 'array<string>',
+	},
+	date: {
+		type: 'string',
+	},
+	urls: {
+		type: 'array<string>',
+	},
+	pages: {
+		type: 'string',
+	},
+	volume: {
+		type: 'string',
+	},
+	number: {
+		type: 'string',
+	},
+	isbn: {
+		type: 'string',
+	},
+	abstract: {
+		type: 'string',
+	},
+	label: {
+		type: 'string',
+	},
+	caption: {
+		type: 'string',
+	},
+	notes: {
+		type: 'string',
+	},
+	address: {
+		type: 'string',
+	},
+	researchNotes: {
+		type: 'string',
+	},
+	keywords: {
+		type: 'array<string>',
+	},
+	accessDate: {
+		type: 'string',
+	},
+	accession: {
+		type: 'string',
+	},
+	doi: {
+		type: 'string',
+	},
+	section: {
+		type: 'string',
+	},
+	language: {
+		type: 'string',
+	},
+	researchNotes: {
+		type: 'string',
+	},
+	databaseProvider: {
+		type: 'string',
+	},
+	database: {
+		type: 'string',
+	},
+	workType: {
+		type: 'string',
+	},
+	custom1: {
+		type: 'string',
+	},
+	custom2: {
+		type: 'string',
+	},
+	custom3: {
+		type: 'string',
+	},
+	custom4: {
+		type: 'string',
+	},
+	custom5: {
+		type: 'string',
+	},
+	custom6: {
+		type: 'string',
+	},
+	custom7: {
+		type: 'string',
+	},
+};

package/lib/formats.js

@@ -0,0 +1,61 @@
+/**
+* Lookup table of various citation file formats
+* @type {array<Object>} A collection of RefLib supported file formats
+* @property {string} title The long form title of the format
+* @property {string} titleShort Shorter title of the format
+* @property {string} input Input format required by parser
+* @property {string} output Output format required by formatter
+* @property {array>string>} ext File extensions of this format, the first entry is generally used as the output default
+* @property {boolean} canRead Whether the format is supported when reading a citation library
+* @property {boolean} canWrite Whether the format is supported when writing a citation library
+*/
+export let formats = {
+	csv: {
+		id: 'csv',
+		title: 'Comma Seperated Values',
+		titleShort: 'CSV',
+		ext: ['.csv'],
+		canRead: false,
+		canWrite: false,
+	},
+	endnoteXml: {
+		id: 'endnoteXml',
+		title: 'EndNoteXML',
+		titleShort: 'EndNoteXML',
+		ext: ['.xml'],
+		canRead: true,
+		canWrite: true,
+	},
+	json: {
+		id: 'json',
+		title: 'JSON',
+		titleShort: 'JSON',
+		ext: ['.json'],
+		canRead: true,
+		canWrite: true,
+	},
+	medline: {
+		id: 'medline',
+		title: 'MEDLINE / PubMed',
+		titleShort: 'MEDLINE',
+		ext: ['.nbib'],
+		canRead: false,
+		canWrite: false,
+	},
+	ris: {
+		id: 'ris',
+		title: 'RIS',
+		titleShort: 'RIS',
+		ext: ['.ris', '.txt'],
+		canRead: true,
+		canWrite: true,
+	},
+	tsv: {
+		id: 'tsv',
+		title: 'Tab Seperated Values',
+		titleShort: 'TSV',
+		ext: ['.tsv'],
+		canRead: false,
+		canWrite: false,
+	},
+}

package/lib/getModule.js

@@ -0,0 +1,37 @@
+import * as modules from '../modules/default.js';
+
+let hasSetup = new Set(); // Modules we have already setup
+
+/**
+* Simple wrapper which loads the named module as a keyed lirary of functions
+* @param {string} module The module ID as per `lib/formats.js`
+* @param {Object} [options] Additional options to use when fetching the module
+* @param {boolean} [options.setup=true] Call the `setup()` function on any module requested before use
+* @return {Object} The loaded module as an object of standardised functionality
+*/
+export function getModule(module, options) {
+	// Sanity checking
+	if (!module) throw new Error('No module provided');
+
+	// Argument mangling
+	let settings = {
+		setup: true,
+		...options,
+	};
+
+	// Try to find the module
+	let mod = modules[module];
+	if (!mod) throw new Error(`Unknown module "${module}"`);
+
+	// Should setup and module exposes a setup function?
+	if (
+		mod.setup // We should set up...
+		&& settings.setup // AND the module has a function to do so...
+		&& !hasSetup.has(module) // AND we've not setup before
+	) {
+		hasSetup.add(module);
+		mod.setup();
+	}
+
+	return mod;
+}

package/lib/identifyFormat.js

@@ -0,0 +1,13 @@
+import {formats} from '../lib/formats.js';
+
+/**
+* Identify and return the Reflib format (from ./formats) to use for the given file name / path
+* @param {string} path The input path to identify
+* @returns {Object} A matching entry from ./formats or null if no matching format was found
+*/
+export function identifyFormat(path) {
+	let ext = /^.*(?<ext>\..+?)$/.exec(path)?.groups.ext.toLowerCase();
+	if (!ext) return null;
+
+	return Object.values(formats).find(format => format.ext.includes(ext));
+}

package/lib/readFile.js

@@ -0,0 +1,34 @@
+import {createReadStream} from 'node:fs';
+import {stat} from 'node:fs/promises';
+import {identifyFormat} from './identifyFormat.js';
+import {readStream} from './readStream.js';
+
+/**
+* Parse a file directly from a path
+* This function is a warpper around the readStream handler + some Promise magic
+* @param {string} path The file path to parse
+* @param {Object} [options] Additional options to pass to the parser
+* @param {string} [options.module] The module to use if overriding from the file path
+* @returns {Promise<Array>} An eventual array of all references parsed from the file
+*/
+export function readFile(path, options) {
+	let settings = {...options};
+	let module = options?.module || identifyFormat(path)?.id;
+	if (!module) throw new Error(`Unable to identify reference library format for file path "${path}"`);
+
+	return stat(path)
+		.then(stats => new Promise((resolve, reject) => {
+			let refs = [];
+			readStream(
+				module,
+				createReadStream(path),
+				{
+					...settings,
+					size: stats.size,
+				},
+			)
+				.on('end', ()=> resolve(refs))
+				.on('error', reject)
+				.on('ref', ref => refs.push(ref))
+		}))
+}

package/lib/readStream.js

@@ -0,0 +1,21 @@
+import {getModule} from './getModule.js';
+
+/**
+* Parse an input stream via a given format ID
+* This function is really just a multiplexor around each modules `readStream` export
+* @param {string} module The module ID as per `lib/formats.js`
+* @param {Stream.Readable} stream Input stream to parse
+* @param {Object} [options] Additional options to pass to the parser
+* @param {number} [options.size] Size of the input stream, if omitted `progress` events are not emitted
+* @returns {EventEmitter} An Event-Emitter compatible object which will fire various events while parsing
+*
+* @emits ref Emitted with an extracted reference object during parse
+* @emits end Emitted when the parsing has completed
+* @emits error Emitted with an Error object if any occured
+*/
+export function readStream(module, stream, options) {
+	if (!module) throw new Error('No module provided to parse with');
+	if (!stream) throw new Error('No stream provided to parse');
+
+	return getModule(module).readStream(stream, options);
+}