citeclaw 2.0.0

package/lib/Translator.js

@@ -0,0 +1,76 @@
+'use strict';
+
+/**
+ * General functions relating to translation
+ * Useable by Scraper.js (html-scraping), JSONReader.js (JSON translation).
+ */
+
+class Translator {
+
+	/**
+	 * Translator constructor
+	 *
+	 * @param {Object} app
+	 */
+	constructor( app ) {
+		this.logger = app.logger;
+	}
+
+	/**
+	 * Adds metadata to citation object given a metadata of a
+	 * specific type, and a translator specific to that metadata type
+	 *
+	 * @param {Object} citation   the citation object
+	 * @param {Object} metadata   flat metadata object (i.e. metadata.openGraph)
+	 * @param {Object} translator
+	 * @param {boolean} creatorOverwrite whether or not unique creator
+	 *                                   translators can be run multiply
+	 * @return {Object}
+	 */
+	translate( citation, metadata, translator, creatorOverwrite ) {
+		const logger = this.logger;
+		if ( !translator || !metadata ) {
+			return citation;
+		}
+		let property;
+		const creatorSubnames = []; // Store types of creators already added
+		let useCreatorTranslator = false; // Bool for whether a creator translator should be used.
+
+		for ( const key in metadata ) {
+			// eslint-disable-next-line no-prototype-builtins
+			if ( metadata.hasOwnProperty( key ) ) { // Loop through properties of metadata Obj
+				// Look up property in the translator to find the
+				// translator function specific to the field
+				property = translator[ key ];
+				if ( !property ) {
+					continue;
+				} // Skip rest of loop if undefined
+				// Set bool to determine if current translator is a
+				// non-duplicate creators translator subname
+				if ( creatorOverwrite && property.name &&
+                        property.name === 'creators' && property.subname ) {
+					// Run translator if not already in array
+					if ( !creatorSubnames.includes( property.subname ) ) {
+						useCreatorTranslator = true;
+						creatorSubnames.push( property.subname ); // i.e. 'author' or 'contributor'
+					}
+				}
+				// Don't overwrite properties already set; only allows
+				// translator to be run only if it's a non-duplicate creator subtype
+				if ( !citation[ property.name ] || useCreatorTranslator ) {
+					try {
+						citation = property.translate( citation, metadata, key );
+						useCreatorTranslator = false; // Set to false for next iteration
+					} catch ( e ) {
+						logger.log( 'debug/translator',
+							`Failed to translate property ${ property.name }` );
+					}
+				}
+			}
+		}
+		return citation;
+	}
+
+}
+
+module.exports = Translator;

package/lib/ZoteroService.js

@@ -0,0 +1,220 @@
+'use strict';
+
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ *
+ * Supplies methods to send requests to a Zotero server
+ */
+
+/* Import Modules */
+const BBPromise = require( 'bluebird' );
+const util = require( 'util' );
+
+const CitoidError = require( './CitoidError.js' );
+const validateZotero = require( './Exporter.js' ).validateZotero;
+const HTTPError = require( './util.js' ).HTTPError;
+
+class ZoteroService {
+
+	/**
+	 * Constructor for CitoidService object
+	 *
+	 * @param {Object} app   Express app; contains logger, metrics, and configuration
+	 */
+	constructor( app ) {
+
+		this.stats = app.metrics;
+
+		this.exporter = null;
+
+		const baseURL = util.format( 'http://%s:%s/',
+			app.conf.zoteroInterface, app.conf.zoteroPort.toString() );
+		this.exportURL = `${ baseURL }export`;
+		this.searchURL = `${ baseURL }search`;
+		this.webURL = `${ baseURL }web`;
+	}
+
+	/**
+	 * Request to Zotero server endpoint /export
+	 *
+	 * @param   {Object}   citation     Zotero JSON citation to be converted
+	 * @param   {Object}   format       requested format
+	 * @return {Object}                 BBPromise for a response
+	 */
+	zoteroExportRequest( citation, format ) {
+		const url = new URL( this.exportURL );
+		url.searchParams.set( 'format', format );
+
+		// eslint-disable-next-line n/no-unsupported-features/node-builtins
+		return BBPromise.resolve( fetch( url.toString(), {
+			method: 'POST',
+			body: JSON.stringify( citation ),
+			headers: {
+				'content-type': 'application/json'
+			}
+		} ) ).then( ( res ) => {
+			// For bibtex format, response is plain text; for other formats, it's JSON
+			const parseResponse = format === 'bibtex' ? res.text() : res.json();
+
+			return parseResponse.then( ( body ) => {
+				if ( !res.ok ) {
+					// Mimic preq HTTPError structure
+					throw new HTTPError( {
+						status: res.status,
+						body: body,
+						message: `HTTP ${ res.status }: ${ res.statusText }`
+					} );
+				}
+				return {
+					status: res.status,
+					body: body
+				};
+			} );
+		} );
+	}
+
+	/**
+	 * Promise for requests to Zotero server endpoint /search
+	 *
+	 * @param  {string}   id       identifier i.e. doi, isbn
+	 * @param  {Object}   cr       CitoidRequest object
+	 * @param  {Object}   citation Citation object
+	 * @return {Object}            BBPromise for response
+	 */
+	zoteroSearchRequest( id, cr, citation ) {
+
+		if ( !id ) {
+			return BBPromise.reject( 'No id provided' );
+		}
+
+		if ( typeof id !== 'string' ) {
+			return BBPromise.reject( 'Invalid type; id must be a string' );
+		}
+		const options = {
+			uri: this.searchURL,
+			method: 'post',
+			headers: {
+				'content-type': 'text/plain'
+			},
+			body: id
+		};
+
+		return cr.request.issueRequest( options ).then( ( response ) => {
+			cr.logger.log( 'debug/zotero', `Zotero search request made for: ${ id }` );
+			this.stats.zotero.req.increment( 1, [ `${ Math.floor( ( response && response.status / 100 ) || 1 ) }xx` ] );
+
+			if ( response && response.status === 200 ) {
+				// Zotero ideally should return 501 if there are no citations in the page,
+				// but occasionally returns empty arrays if it successfully locates a
+				// translator but the translator returns no metadata.
+				if ( response.body && Array.isArray( response.body ) && response.body[ 0 ] ) {
+					// Case where response is an Array inside of an Array;
+					if ( Array.isArray( response.body[ 0 ] ) ) {
+						if ( response.body[ 0 ][ 0 ] ) {
+							// Rewrites response.body to be an Array of objects
+							response.body = response.body[ 0 ];
+						} else { // Case where body is [[]]
+							return BBPromise.reject( 'No citation in body' );
+						}
+					}
+
+					// Case where body is an empty object, i.e. [{}] or [[{}]]
+					if ( !Object.keys( response.body[ 0 ] ).length ) {
+						return BBPromise.reject( 'No citation in body' );
+					}
+
+					citation.content = response.body[ 0 ];
+
+					// Validate citation
+					citation.content = validateZotero( null, citation.content );
+					citation.source.push( 'Zotero' );
+
+					return cr;
+				} else {
+					return BBPromise.reject( 'No citation in body' );
+				}
+			} else { // I.e. 300 response codes
+				return BBPromise.reject( 'Non 200 response from Zotero' );
+			}
+		}, ( response ) => {
+			cr.logger.log( 'debug/zotero', `Zotero request made for: ${ id }` );
+			this.stats.zotero.req.increment( 1, [ `${ Math.floor( ( response && response.status / 100 ) || 1 ) }xx` ] );
+			return BBPromise.reject( response );
+		} );
+	}
+
+	/**
+	 * Promise for requests to Zotero server endpoint /web
+	 *
+	 * @param  {Object}   cr       CitoidRequest object
+	 * @param  {Object}   citation Citation object
+	 * @return {Object}            BBPromise for response
+	 */
+	zoteroWebRequest( cr, citation ) {
+
+		const requestedURL = citation.resolvedUrl || citation.url; // Prefer resolved
+
+		if ( !requestedURL ) {
+			return BBPromise.reject( 'No url in Citation object' );
+		}
+
+		const options = {
+			uri: this.webURL,
+			method: 'post',
+			headers: {
+				'content-type': 'text/plain'
+			},
+			body: requestedURL
+		};
+
+		return cr.request.issueRequest( options ).then( ( response ) => {
+			cr.logger.log( 'debug/zotero', `Zotero web request made for: ${ requestedURL }` );
+			this.stats.zotero.req.increment( 1, [ `${ Math.floor( ( response && response.status / 100 ) || 1 ) }xx` ] );
+
+			if ( response && response.status === 200 ) {
+				// Zotero ideally should return 501 if there are no citations in the page,
+				// but occasionally returns empty arrays if it successfully locates a
+				// translator but the translator returns no metadata.
+				if ( response.body && Array.isArray( response.body ) && response.body[ 0 ] ) {
+					// Case where response is an Array inside of an Array;
+					if ( Array.isArray( response.body[ 0 ] ) ) {
+						if ( response.body[ 0 ][ 0 ] ) {
+							// Rewrites response.body to be an Array of objects
+							response.body = response.body[ 0 ];
+						} else { // Case where body is [[]]
+							return BBPromise.reject( 'No citation in body' );
+						}
+					}
+
+					// Case where body is an empty object, i.e. [{}] or [[{}]]
+					if ( !Object.keys( response.body[ 0 ] ).length ) {
+						return BBPromise.reject( 'No citation in body' );
+					}
+
+					citation.content = response.body[ 0 ];
+
+					// Validate citation
+					citation.content = validateZotero( requestedURL, citation.content );
+					citation.source.push( 'Zotero' );
+
+					return cr;
+				} else {
+					return BBPromise.reject( 'No citation in body' );
+				}
+			} else { // I.e. 300 response codes
+				return BBPromise.reject( 'Non 200 response from Zotero' );
+			}
+		}, ( response ) => {
+			cr.logger.log( 'debug/zotero', `Zotero web request failed for ${ requestedURL }` );
+			if ( response.status === 400 && response.body && response.body.detail === 'The remote document is not in a supported format' ) {
+				cr.error = new CitoidError( null, response.body.detail, 415 );
+			}
+			this.stats.zotero.req.increment( 1, [ `${ Math.floor( ( response && response.status / 100 ) || 1 ) }xx` ] );
+			return BBPromise.reject( response );
+		} );
+	}
+
+}
+
+/* Exports */
+module.exports = ZoteroService;

package/lib/api-util.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const BBPromise = require( 'bluebird' );
+const sUtil = require( './util' );
+const Template = require( 'swagger-router' ).Template;
+const HTTPError = sUtil.HTTPError;
+
+/**
+ * Calls the MW API with the supplied query as its body
+ *
+ * @param {!Object} req the incoming request object
+ * @param {?Object} query an object with all the query parameters for the MW API
+ * @param {?Object} headers additional headers to pass to the MW API
+ * @return {!Promise} a promise resolving as the response object from the MW API
+ */
+function mwApiGet( req, query, headers ) {
+
+	const app = req.app;
+	query = Object.assign( {
+		format: 'json',
+		formatversion: 2
+	}, query );
+
+	const request = app.mwapi_tpl.expand( {
+		request: {
+			params: { domain: req.params.domain },
+			headers: req.headers,
+			query
+		}
+	} );
+	Object.assign( request.headers, headers );
+
+	return req.issueRequest( request ).then( ( response ) => {
+		if ( response.status < 200 || response.status > 399 ) {
+			// there was an error when calling the upstream service, propagate that
+			return BBPromise.reject( new HTTPError( {
+				status: response.status,
+				type: 'api_error',
+				title: 'MW API error',
+				detail: response.body
+			} ) );
+		}
+		return response;
+	} );
+
+}
+
+/**
+ * Calls the REST API with the supplied domain, path and request parameters
+ *
+ * @param {!Object} req the incoming request object
+ * @param {?string} path the REST API path to contact without the leading slash
+ * @param {?Object} [restReq={}] the object containing the REST request details
+ * @param {?string} [restReq.method=get] the request method
+ * @param {?Object} [restReq.query={}] the query string to send, if any
+ * @param {?Object} [restReq.headers={}] the request headers to send
+ * @param {?Object} [restReq.body=null] the body of the request, if any
+ * @return {!Promise} a promise resolving as the response object from the REST API
+ */
+function restApiGet( req, path, restReq ) {
+
+	const app = req.app;
+	if ( path.constructor === Object ) {
+		restReq = path;
+		path = undefined;
+	}
+	restReq = restReq || {};
+	restReq.method = restReq.method || 'get';
+	restReq.query = restReq.query || {};
+	restReq.headers = restReq.headers || {};
+	restReq.params = restReq.params || {};
+	restReq.params.path = path || restReq.params.path;
+	restReq.params.domain = restReq.params.domain || req.params.domain;
+	if ( !restReq.params.path || !restReq.params.domain ) {
+		return BBPromise.reject( new HTTPError( {
+			status: 500,
+			type: 'internal_error',
+			title: 'Invalid internal call',
+			detail: 'domain and path need to be defined for the REST API call'
+		} ) );
+	}
+	restReq.params.path = restReq.params.path[ 0 ] === '/' ?
+		restReq.params.path.slice( 1 ) : restReq.params.path;
+
+	return req.issueRequest( app.restbase_tpl.expand( { request: restReq } ) );
+
+}
+
+/**
+ * Sets up the request templates for MW and RESTBase API requests
+ *
+ * @param {!Application} app the application object
+ */
+function setupApiTemplates( app ) {
+
+	// set up the MW API request template
+	if ( !app.conf.mwapi_req ) {
+		app.conf.mwapi_req = {
+			method: 'post',
+			uri: 'http://{{domain}}/w/api.php',
+			headers: '{{request.headers}}',
+			body: '{{ default(request.query, {}) }}'
+		};
+	}
+	app.mwapi_tpl = new Template( app.conf.mwapi_req );
+
+	// set up the RESTBase request template
+	if ( !app.conf.restbase_req ) {
+		app.conf.restbase_req = {
+			method: '{{request.method}}',
+			uri: 'http://{{domain}}/api/rest_v1/{+path}',
+			query: '{{ default(request.query, {}) }}',
+			headers: '{{request.headers}}',
+			body: '{{request.body}}'
+		};
+	}
+	app.restbase_tpl = new Template( app.conf.restbase_req );
+
+}
+
+module.exports = {
+	mwApiGet,
+	restApiGet,
+	setupApiTemplates
+};

package/lib/external-apis/CrossRefService.js

@@ -0,0 +1,122 @@
+'use strict';
+
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ *
+ * Supplies methods to send requests to the CrossRef REST API
+ * https://github.com/CrossRef/rest-api-doc
+ */
+
+/* Import Modules */
+const BBPromise = require( 'bluebird' );
+const url = 'https://api.crossref.org/works/';
+
+class CrossRefService {
+
+	/**
+	 * Constructor for CrossRefService object
+	 *
+	 * @param {Object} app   Express app; contains configuration
+	 */
+	constructor( app ) {
+
+		this.mailto = app.conf.mailto;
+
+	}
+
+	/**
+	 * Request crossRef metadata from API via
+	 *
+	 * @param  {string}    doi     doi
+	 * @param  {Object}    request original request object
+	 * @return {BBPromise}         Promise for metadata from CrossRef
+	 */
+	doi( doi, request ) {
+		request.logger.log( 'debug/other', 'Making request to CrossRef REST API works with doi' );
+
+		if ( !doi || typeof doi !== 'string' ) {
+			BBPromise.reject( 'No DOI in argument' );
+		}
+
+		const encodedDOI = encodeURIComponent( doi );
+		const doiUrl = url + encodedDOI;
+		let qs = null;
+
+		// Suppling mailto as a query parameter gets us the "polite" level of service,
+		// which has better reliability. See:
+		// https://github.com/CrossRef/rest-api-doc#meta
+		if ( this.mailto ) {
+			qs = { mailto: this.mailto };
+		}
+
+		const requestOptions = {
+			uri: doiUrl,
+			headers: {
+				'User-Agent': this.userAgent
+			},
+			qs,
+			qsStringifyOptions: { // Prevent email from being URL encoded
+				encode: false
+			}
+		};
+
+		// Make request to crossref
+		return request.issueRequest( requestOptions ).then( ( res ) => {
+			if ( res && res.status === 200 && res.body.status && res.body.status === 'ok' ) {
+				return res.body.message;
+			} else {
+				return BBPromise.reject( `No results for doi ${ doi }` );
+			}
+		} );
+
+	}
+
+	/**
+	 * Request crossRef metadata from API via any input
+	 *
+	 * @param  {string}    any     any part of a citation, i.e. title or full citation
+	 * @param  {Object}    request original request object
+	 * @return {BBPromise}         Promise for metadata from CrossRef
+	 */
+	search( any, request ) {
+		request.logger.log( 'debug/other', 'Making request to CrossRef REST API with search query' );
+
+		any = encodeURIComponent( any );
+
+		const qs = {
+			query: any,
+			rows: 2 // Request two results
+		};
+
+		// Suppling mailto as a query parameter gets us the "polite" level of service,
+		// which has better reliability. See:
+		// https://github.com/CrossRef/rest-api-doc#meta
+		if ( this.mailto ) {
+			qs.mailto = this.mailto;
+		}
+
+		const requestOptions = {
+			uri: url,
+			qs,
+			qsStringifyOptions: { // Prevent email from being URL encoded
+				encode: false
+			}
+		};
+
+		// Make request to crossref
+		return request.issueRequest( requestOptions ).then( ( res ) => {
+			if ( res && res.status === 200 && res.body.status && res.body.status === 'ok' &&
+                res.body[ 'message-type' ] === 'work-list' && res.body.message &&
+                res.body.message.items && Array.isArray( res.body.message.items ) &&
+                res.body.message.items.length > 0 ) {
+				return res.body.message.items.slice( 0, 2 ); // Return up to two results
+			} else {
+				return BBPromise.reject( `No results for search string ${ any }` );
+			}
+		} );
+
+	}
+
+}
+
+module.exports = CrossRefService;