citeclaw 2.0.0

package/lib/util.js

@@ -0,0 +1,410 @@
+'use strict';
+
+const BBPromise = require( 'bluebird' );
+const express = require( 'express' );
+const { v1: uuidv1 } = require( 'uuid' );
+const bunyan = require( 'bunyan' );
+const preq = require( 'preq' );
+
+/**
+ * Error instance wrapping HTTP error responses
+ */
+class HTTPError extends Error {
+
+	constructor( response ) {
+		super();
+		Error.captureStackTrace( this, HTTPError );
+
+		if ( response.constructor !== Object ) {
+			// just assume this is just the error message
+			response = {
+				status: 500,
+				type: 'internal_error',
+				title: 'InternalError',
+				detail: response
+			};
+		}
+
+		this.name = this.constructor.name;
+		this.message = `${ response.status }`;
+		if ( response.type ) {
+			this.message += `: ${ response.type }`;
+		}
+
+		Object.assign( this, response );
+	}
+}
+
+/**
+ * Generates an object suitable for logging out of a request object
+ *
+ * @param {!Request} req          the request
+ * @param {?RegExp}  whitelistRE  the RegExp used to filter headers
+ * @return {!Object} an object containing the key components of the request
+ */
+function reqForLog( req, whitelistRE ) {
+
+	const params = {};
+
+	// Though not actually a query parameter, backwards compatibility with older logs
+	if ( req.params && req.params.catchall && req.params.catchall[ 0 ] ) {
+		params.format = req.params.catchall[ 0 ];
+	}
+
+	const ret = {
+		url: req.originalUrl,
+		headers: {},
+		method: req.method,
+		params: params,
+		query: req.query,
+		body: req.body,
+		remoteAddress: req.connection.remoteAddress,
+		remotePort: req.connection.remotePort
+	};
+
+	if ( req.headers && whitelistRE ) {
+		Object.keys( req.headers ).forEach( ( hdr ) => {
+			if ( whitelistRE.test( hdr ) ) {
+				ret.headers[ hdr ] = req.headers[ hdr ];
+			}
+		} );
+	}
+
+	return ret;
+
+}
+
+/**
+ * serializes an uri in a form suitable for logging.
+ *
+ * @param {string} uri in string form to serialize
+ * @return {!Object} the serialized version of the uri
+ */
+function uriForLog( uri ) {
+	const serializedURI = {
+		uri: null,
+		hostname: null
+	};
+
+	if ( !uri || typeof uri !== 'string' ) {
+		return serializedURI;
+	}
+
+	serializedURI.uri = uri;
+
+	try {
+		serializedURI.hostname = new URL( uri ).hostname;
+	} catch ( e ) {
+	}
+
+	return serializedURI;
+}
+
+/**
+ * Serializes an error object in a form suitable for logging.
+ *
+ * @param {!Error} err error to serialize
+ * @return {!Object} the serialized version of the error
+ */
+function errForLog( err ) {
+
+	const ret = bunyan.stdSerializers.err( err );
+
+	ret.status = err.status;
+	ret.type = err.type;
+	ret.detail = err.detail;
+
+	// Now supplanted by outgoingReqResult
+	if ( err.body && err.body.internalMethod ) {
+		ret.method = err.body.internalMethod;
+	}
+
+	// Now supplanted by outgoingReqResult
+	if ( err.body && err.body.internalURI ) {
+		ret.uri = uriForLog( err.body.internalURI );
+	}
+
+	// remove the stack trace only for 500 errors and if enabled
+	if ( Number.parseInt( ret.status, 10 ) !== 500 ) {
+		ret.stack = undefined;
+	}
+
+	return ret;
+
+}
+
+/**
+ * Serializes an response object in a form suitable for logging
+ * from outgoing requests. Since these are HTTP errors from
+ * other sites, we don't need the stack trace.
+ *
+ * @param {!Object} res  custom obj with status, uri, or error
+ * @return {!Object} the serialized version of the error
+ */
+function outgoingReqResult( res ) {
+	const ret = {};
+	if ( !res ) {
+		return ret;
+	}
+	if ( res.status && typeof res.status === 'number' ) {
+		ret.status = res.status;
+	}
+
+	// Log archived urls
+	if ( res.archive && typeof res.archive === 'string' ) {
+		ret.archive = res.archive;
+	}
+
+	// Don't include full error to avoid logging json from error.msg
+	if ( res.error && res.error instanceof Error ) {
+		const error = errForLog( res.error );
+		ret.status = error.status;
+		ret.error = error.name;
+		if ( error.uri ) {
+			ret.hostname = error.uri.hostname;
+			ret.uri = error.uri.uri;
+		}
+	}
+	if ( res.uri ) {
+		const uri = uriForLog( res.uri );
+		if ( uri.hostname ) {
+			ret.hostname = uri.hostname;
+		}
+		if ( uri.uri ) {
+			ret.uri = uri.uri;
+		}
+	}
+	return ret;
+}
+
+/**
+ * Serializes an CitoidResponse object in a form suitable for logging,
+ * but requires the full CitoidRequest object as input
+ *
+ * @param {!Object} cReq CitoidRequest to serialize
+ * @return {!Object} the serialized version of the error
+ */
+function crForLog( cReq ) {
+
+	const ret = {};
+	if ( !cReq ) {
+		return ret;
+	}
+
+	// ResponseCode we send to users
+	if ( cReq.response ) {
+		ret.responseCode = cReq.response.responseCode;
+		ret.body = cReq.response.body;
+
+		// Gets HTTPError from CitoidError object
+		if ( cReq.response.error && cReq.response.error.error ) {
+			ret.error = errForLog( cReq.response.error.error, true ); // Don't log stacktrace
+		}
+	}
+
+	return ret;
+
+}
+
+/**
+ * Wraps all of the given router's handler functions with
+ * promised try blocks so as to allow catching all errors,
+ * regardless of whether a handler returns/uses promises
+ * or not.
+ *
+ * @param {!Object} route the object containing the router and path to bind it to
+ * @param {!Application} app the application object
+ */
+function wrapRouteHandlers( route, app ) {
+
+	route.router.stack.forEach( ( routerLayer ) => {
+		const path = ( route.path + routerLayer.route.path.slice( 1 ) )
+			.replace( /\/:/g, '/--' )
+			.replace( /^\//, '' )
+			.replace( /[/?]+$/, '' );
+		routerLayer.route.stack.forEach( ( layer ) => {
+			const origHandler = layer.handle;
+			const metric = app.metrics.makeMetric( {
+				type: 'Histogram',
+				name: 'router',
+				prometheus: {
+					name: 'citoid_router_request_duration_seconds',
+					help: 'request duration handled by router in seconds',
+					staticLabels: app.metrics.getServiceLabel(),
+					buckets: [ 0.01, 0.05, 0.1, 0.3, 1, 30, 120 ]
+				},
+				labels: {
+					names: [ 'path', 'method', 'status' ],
+					omitLabelNames: true
+				}
+			} );
+			layer.handle = ( req, res, next ) => {
+				const startTime = Date.now();
+				BBPromise.try( () => origHandler( req, res, next ) )
+					.catch( next )
+					.finally( () => {
+						let statusCode = parseInt( res.statusCode, 10 ) || 500;
+						if ( statusCode < 100 || statusCode > 599 ) {
+							statusCode = 500;
+						}
+						metric.endTiming( startTime, [ path || 'root', req.method, statusCode ] );
+					} );
+			};
+		} );
+	} );
+
+}
+
+/**
+ * Generates an error handler for the given applications and installs it.
+ *
+ * @param {!Application} app the application object to add the handler to
+ */
+function setErrorHandler( app ) {
+
+	app.use( ( err, req, res, next ) => {
+		let errObj;
+		// ensure this is an HTTPError object
+		if ( err.constructor === HTTPError ) {
+			errObj = err;
+		} else if ( err instanceof Error ) {
+			// is this an HTTPError defined elsewhere? (preq)
+			if ( err.constructor.name === 'HTTPError' ) {
+				const o = { status: err.status };
+				if ( err.body && err.body.constructor === Object ) {
+					Object.keys( err.body ).forEach( ( key ) => {
+						o[ key ] = err.body[ key ];
+					} );
+				} else {
+					o.detail = err.body;
+				}
+				o.message = err.message;
+				errObj = new HTTPError( o );
+			} else {
+				// this is a standard error, convert it
+				errObj = new HTTPError( {
+					status: 500,
+					type: 'internal_error',
+					title: err.name,
+					detail: err.detail || err.message,
+					stack: err.stack
+				} );
+			}
+		} else if ( err.constructor === Object ) {
+			// this is a regular object, suppose it's a response
+			errObj = new HTTPError( err );
+		} else {
+			// just assume this is just the error message
+			errObj = new HTTPError( {
+				status: 500,
+				type: 'internal_error',
+				title: 'InternalError',
+				detail: err
+			} );
+		}
+		// ensure some important error fields are present
+		errObj.status = errObj.status || 500;
+		errObj.type = errObj.type || 'internal_error';
+		// add the offending URI and method as well
+		errObj.method = errObj.method || req.method;
+		errObj.uri = errObj.uri || req.url;
+		// some set 'message' or 'description' instead of 'detail'
+		errObj.detail = errObj.detail || errObj.message || errObj.description || '';
+		// adjust the log level based on the status code
+		let level = 'error';
+		if ( Number.parseInt( errObj.status, 10 ) < 400 ) {
+			level = 'trace';
+		} else if ( Number.parseInt( errObj.status, 10 ) < 500 ) {
+			level = 'info';
+		}
+		// log the error
+		const component = ( errObj.component ? errObj.component : errObj.status );
+		( req.logger || app.logger ).log( `${ level }/${ component }`, errForLog( errObj ) );
+		// let through only non-sensitive info
+		const respBody = {
+			status: errObj.status,
+			type: errObj.type,
+			title: errObj.title,
+			detail: errObj.detail,
+			method: errObj.method,
+			uri: errObj.uri
+		};
+		res.status( errObj.status ).json( respBody );
+	} );
+
+}
+
+/**
+ * Creates a new router with some default options.
+ *
+ * @param {?Object} [opts] additional options to pass to express.Router()
+ * @return {!Router} a new router object
+ */
+function createRouter( opts ) {
+
+	const options = {
+		mergeParams: true
+	};
+
+	if ( opts && opts.constructor === Object ) {
+		Object.assign( options, opts );
+	}
+
+	return new express.Router( options );
+
+}
+
+/**
+ * Adds logger to the request and logs it.
+ *
+ * @param {!*} req request object
+ * @param {!Application} app application object
+ */
+function initAndLogRequest( req, app ) {
+	req.headers = req.headers || {};
+	req.headers[ 'x-request-id' ] = req.headers[ 'x-request-id' ] || uuidv1();
+	req.logger = app.logger.child( {
+		request_id: req.headers[ 'x-request-id' ],
+		request: reqForLog( req, app.conf.log_header_whitelist )
+	} );
+	req.context = { reqId: req.headers[ 'x-request-id' ] };
+	req.issueRequest = ( request ) => {
+		if ( !( request.constructor === Object ) ) {
+			request = { uri: request };
+		}
+		if ( request.url ) {
+			request.uri = request.url;
+			delete request.url;
+		}
+		if ( !request.uri ) {
+			return BBPromise.reject( new HTTPError( {
+				status: 500,
+				type: 'internal_error',
+				title: 'No request to issue',
+				detail: 'No request has been specified'
+			} ) );
+		}
+		request.method = request.method || 'get';
+		request.headers = request.headers || {};
+		Object.assign( request.headers, {
+			'accept-language': req.headers[ 'accept-language' ], // modification from service-template-node - pass through accept-language header
+			'user-agent': app.conf.user_agent,
+			'x-request-id': req.context.reqId
+		} );
+		req.logger.log( 'trace/req', { msg: 'Outgoing request', out_request: request } );
+		return app.outgoingRequestThrottle.schedule( request.uri, () => preq( request ) );
+	};
+	req.logger.log( 'trace/req', { msg: 'Incoming request' } );
+}
+
+module.exports = {
+	HTTPError,
+	crForLog,
+	errForLog,
+	outgoingReqResult,
+	uriForLog,
+	initAndLogRequest,
+	wrapRouteHandlers,
+	setErrorHandler,
+	router: createRouter
+};

package/lib/zotero/cachedTypes.js

@@ -0,0 +1,232 @@
+/**
+ * This file is modified from a file copied from Zotero below
+ */
+
+/*
+    ***** BEGIN LICENSE BLOCK *****
+
+    Copyright © 2011 Center for History and New Media
+                     George Mason University, Fairfax, Virginia, USA
+                     http://zotero.org
+
+    This file is part of Zotero.
+
+    Zotero is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Zotero is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
+
+    ***** END LICENSE BLOCK *****
+*/
+
+/* eslint-disable */
+'use strict';
+
+/**
+ * Emulates very small parts of cachedTypes.js and itemFields.js APIs for use with connector
+ */
+const schema = require('./typeSchemaData.js');
+const extend = require('extend');
+/**
+ * @namespace
+ */
+function Zotero() {
+    this.schemaTypes = ["itemTypes", "creatorTypes", "fields"];
+    this.schema = extend(true, {}, schema); // Deep copy schema
+    this.uncachedSchema = schema; // Keep original schema
+
+    // attach IDs and make referenceable by either ID or name
+    for (let i = 0; i < this.schemaTypes.length; i++) {
+        const schemaType = this.schemaTypes[i];
+        for (const id in this.schema[schemaType]) {
+            const entry = this.schema[schemaType][id];
+            entry.unshift(parseInt(id, 10));
+            this.schema[schemaType][entry[1]/* name */] = entry;
+        }
+    }
+
+    this.itemTypes = this.schema.itemTypes;
+    this.creatorTypes = this.schema.creatorTypes;
+    this.fields = this.schema.fields;
+
+    this.creatorTypesMethods = new CreatorTypes(this.creatorTypes, this.itemTypes);
+    this.itemFieldsMethods = new ItemFields(this.fields, this.itemTypes);
+    this.itemTypeMethods = new CachedTypes(this.itemTypes);
+}
+
+/**
+ * Gets object containing names of basefields for a given itemType
+ * @param  {String} typeIdOrName itemType- in ID or Name
+ * @return {Object}              typemap in {field:basefield} format
+ */
+Zotero.prototype.getBaseFields = function(typeIdOrName) {
+    const self = this;
+    let baseField;
+    let field;
+    const baseFieldObj = this.itemTypes[typeIdOrName][5];
+    const creatorsList = this.itemTypes[typeIdOrName][3];
+    const firstCreator = creatorsList[0];
+    const typeMap = {};
+
+    // Regular base fields
+    Object.keys(baseFieldObj).forEach((itemTypeID) => { // Loop through all item types
+        field = self.itemFieldsMethods.cachedTypeMethods.getName(itemTypeID);
+        baseField = self.itemFieldsMethods.cachedTypeMethods.getName(baseFieldObj[itemTypeID]);
+        typeMap[field] = baseField;
+    });
+
+    // Creator base fields
+    if (firstCreator && firstCreator !== 1) { // If first creator in the list is not author
+        field = self.creatorTypesMethods.cachedTypeMethods.getName(firstCreator);
+        baseField = self.creatorTypesMethods.cachedTypeMethods.getName(1); // Basefield is always author
+        typeMap[field] = baseField;
+    }
+
+    if (Object.keys(typeMap).length > 0) {
+        return typeMap;
+    } else {
+        return false;
+    }
+};
+
+
+/**
+ * Request info from a cached type
+ * @param {Object} schemaByType type specific schema segment
+ */
+function CachedTypes(schemaByType) {
+    this.thisType = schemaByType;
+}
+
+CachedTypes.prototype.getID = function(idOrName) {
+    const type = this.thisType[idOrName];
+    return (type ? type[0]/* id */ : false);
+};
+
+CachedTypes.prototype.getName = function(idOrName) {
+    const type = this.thisType[idOrName];
+    return (type ? type[1]/* name */ : false);
+};
+
+CachedTypes.prototype.getLocalizedString = function(idOrName) {
+    const type = this.thisType[idOrName];
+    return (type ? type[2]/* localizedString */ : false);
+};
+
+
+function CreatorTypes(creatorTypes, itemTypes) {
+    this.schemaType = "creatorTypes";
+    this.itemTypes = itemTypes;
+    this.creatorTypes = creatorTypes;
+    this.cachedTypeMethods = new CachedTypes(creatorTypes);
+}
+
+CreatorTypes.prototype.getTypesForItemType = function(idOrName) {
+    const itemType = this.itemTypes[idOrName];
+    if (!itemType) { return false; }
+
+    let itemCreatorTypes = itemType[3]/* creatorTypes */,
+        n = itemCreatorTypes.length,
+        outputTypes = [];
+
+    for (let i = 0; i < n; i++) {
+        const creatorType = this.creatorTypes[itemCreatorTypes[i]];
+        if (creatorType) {
+            outputTypes.push( {
+                "id":creatorType[0]/* id */,
+                "name":creatorType[1]/* name */ 
+            } );
+        }
+    }
+    return outputTypes;
+};
+
+CreatorTypes.prototype.getPrimaryIDForType = function(idOrName) {
+    const itemType = this.itemTypes[idOrName];
+    if (!itemType) { return false; }
+    return itemType[3]/* creatorTypes */[0];
+};
+
+CreatorTypes.prototype.isValidForType = function(fieldIdOrName, typeIdOrName) {
+    const field = this.creatorTypes[fieldIdOrName];
+    const itemType = this.itemTypes[typeIdOrName];
+    // mimics itemFields.js
+    if (!field || !itemType) { return false; }
+
+    /* fields */        /* id */
+    return itemType[3].indexOf(field[0]) !== -1;
+};
+
+function ItemFields(fields, itemTypes) {
+    this.schemaType = "fields";
+    this.fields = fields;
+    this.itemTypes = itemTypes;
+    this.cachedTypeMethods = new CachedTypes(fields);
+}
+
+ItemFields.prototype.isValidForType = function(fieldIdOrName, typeIdOrName) {
+    const field = this.fields[fieldIdOrName];
+    const itemType = this.itemTypes[typeIdOrName];
+
+    // whitelist itemType
+    if (fieldIdOrName === 'itemType') {
+        return true;
+    }
+
+    // whitelist creators field when there are valid creator names
+    // Only exception is note itemType
+    if (fieldIdOrName === 'creators' && itemType[3].length > 0 && itemType !== 'note') {
+        return true;
+    }
+
+    // mimics itemFields.js
+    if (!field || !itemType) { return false; }
+
+    /* fields */        /* id */
+    return itemType[4].indexOf(field[0]) !== -1;
+};
+
+ItemFields.prototype.getFieldIDFromTypeAndBase = function(typeIdOrName, fieldIdOrName) {
+    let baseField = this.fields[fieldIdOrName],
+        itemType = this.itemTypes[typeIdOrName];
+
+    if (!baseField || !itemType) { return false; }
+
+    // get as ID
+    baseField = baseField[0]/* id */;
+
+    // loop through base fields for item type
+    const baseFields = itemType[5];
+    for (const i in baseFields) {
+        if (baseFields[i] === baseField) {
+            return parseInt(i);
+        }
+    }
+
+    return false;
+};
+
+ItemFields.prototype.getBaseIDFromTypeAndField = function(typeIdOrName, fieldIdOrName) {
+    let field = this.fields[fieldIdOrName],
+        itemType = this.itemTypes[typeIdOrName];
+    if (!field || !itemType) {
+        throw new Error("Invalid field or type ID");
+    }
+
+    const baseField = itemType[5]/* baseFields */[field[0]/* id */];
+    return baseField || false;
+};
+
+ItemFields.prototype.getItemTypeFields = function(typeIdOrName) {
+    return this.itemTypes[typeIdOrName][4]/* fields */.slice();
+};
+
+module.exports = Zotero;