citeclaw 2.0.0

package/lib/external-apis/OpenAlexService.js

@@ -0,0 +1,245 @@
+'use strict';
+
+const BBPromise = require( 'bluebird' );
+
+const { validateZotero } = require( '../Exporter.js' );
+const cRef = require( '../translators/crossRef.js' );
+
+class OpenAlexService {
+
+	constructor( app ) {
+		const conf = app.conf.openAlex || {};
+		this.api = conf.api || 'https://api.openalex.org';
+		this.apiKey = conf.apiKey || null;
+		this.enabled = !!conf.enabled;
+		this.searchLimit = Number.isFinite( conf.searchLimit ) ? conf.searchLimit : 2;
+		this.requestTimeout = Number.isFinite( conf.timeout ) ? conf.timeout : 5000;
+	}
+
+	issueRequest( request, options ) {
+		const qs = Object.assign( {}, options.qs );
+		if ( this.apiKey ) {
+			qs.api_key = this.apiKey;
+		}
+
+		return request.issueRequest( {
+			uri: `${ this.api }${ options.path }`,
+			qs,
+			timeout: this.requestTimeout
+		} );
+	}
+
+	getWork( identifier, request ) {
+		if ( !identifier ) {
+			return BBPromise.reject( 'No OpenAlex identifier provided' );
+		}
+
+		request.logger.log( 'debug/other', `Making request to OpenAlex for work ${ identifier }` );
+
+		return this.issueRequest( request, {
+			path: `/works/${ encodeURIComponent( identifier ) }`
+		} ).then( ( response ) => {
+			if ( response && response.status === 200 && response.body && response.body.id ) {
+				return response.body;
+			}
+			return BBPromise.reject( `No OpenAlex work found for ${ identifier }` );
+		} );
+	}
+
+	searchWorks( query, request ) {
+		if ( !query ) {
+			return BBPromise.reject( 'No OpenAlex search query provided' );
+		}
+
+		request.logger.log( 'debug/other', `Making request to OpenAlex search for ${ query }` );
+
+		return this.issueRequest( request, {
+			path: '/works',
+			qs: {
+				search: query,
+				'per-page': this.searchLimit
+			}
+		} ).then( ( response ) => {
+			if ( response && response.status === 200 && response.body &&
+				Array.isArray( response.body.results ) && response.body.results.length ) {
+				return response.body.results.slice( 0, this.searchLimit );
+			}
+			return BBPromise.reject( `No OpenAlex results found for ${ query }` );
+		} );
+	}
+
+	toCitation( work, originalQuery ) {
+		if ( !work || !work.id || !work.display_name ) {
+			throw new Error( 'Invalid OpenAlex work payload' );
+		}
+
+		const doiUrl = work.doi || work.ids && work.ids.doi || null;
+		const doi = this.normalizeDoi( doiUrl );
+		const citation = {
+			itemType: this.resolveItemType( work ),
+			title: work.display_name,
+			url: this.resolveUrl( work, originalQuery ),
+			libraryCatalog: 'OpenAlex',
+			accessDate: ( new Date() ).toISOString().slice( 0, 10 )
+		};
+		const abstract = this.restoreAbstract( work.abstract_inverted_index );
+		const source = work.primary_location && work.primary_location.source;
+		const biblio = work.biblio || {};
+		const pages = this.buildPages( biblio.first_page, biblio.last_page );
+		const extra = this.buildExtra( work );
+
+		if ( doi ) {
+			citation.DOI = doi;
+		}
+		if ( abstract ) {
+			citation.abstractNote = abstract;
+		}
+		if ( work.publication_date ) {
+			citation.date = work.publication_date;
+		} else if ( work.publication_year ) {
+			citation.date = String( work.publication_year );
+		}
+		if ( source && source.display_name ) {
+			citation.publicationTitle = source.display_name;
+		}
+		if ( biblio.volume ) {
+			citation.volume = String( biblio.volume );
+		}
+		if ( biblio.issue ) {
+			citation.issue = String( biblio.issue );
+		}
+		if ( pages ) {
+			citation.pages = pages;
+		}
+		if ( Array.isArray( work.authorships ) && work.authorships.length ) {
+			citation.creators = work.authorships
+				.map( ( authorship ) => this.authorToCreator( authorship ) )
+				.filter( Boolean );
+		}
+		if ( extra ) {
+			citation.extra = extra;
+		}
+
+		return validateZotero( null, citation );
+	}
+
+	resolveItemType( work ) {
+		if ( work.type_crossref && cRef.types[ work.type_crossref ] ) {
+			return cRef.types[ work.type_crossref ];
+		}
+
+		const mapping = {
+			article: 'journalArticle',
+			book: 'book',
+			'book-chapter': 'bookSection',
+			dataset: 'webpage',
+			dissertation: 'thesis',
+			editorial: 'journalArticle',
+			erratum: 'journalArticle',
+			letter: 'journalArticle',
+			monograph: 'book',
+			'peer-review': 'journalArticle',
+			preprint: 'preprint',
+			'proceedings-article': 'conferencePaper',
+			report: 'report',
+			review: 'journalArticle',
+			standard: 'document'
+		};
+
+		return mapping[ work.type ] || 'journalArticle';
+	}
+
+	resolveUrl( work, originalQuery ) {
+		if ( work.doi ) {
+			return work.doi;
+		}
+		if ( work.primary_location && work.primary_location.landing_page_url ) {
+			return work.primary_location.landing_page_url;
+		}
+		if ( work.best_oa_location && work.best_oa_location.landing_page_url ) {
+			return work.best_oa_location.landing_page_url;
+		}
+		if ( typeof originalQuery === 'string' && originalQuery.startsWith( 'http' ) ) {
+			return originalQuery;
+		}
+		return work.id;
+	}
+
+	restoreAbstract( invertedIndex ) {
+		if ( !invertedIndex || typeof invertedIndex !== 'object' ) {
+			return null;
+		}
+
+		const words = [];
+		Object.keys( invertedIndex ).forEach( ( token ) => {
+			const positions = invertedIndex[ token ];
+			if ( Array.isArray( positions ) ) {
+				positions.forEach( ( position ) => {
+					words[ position ] = token;
+				} );
+			}
+		} );
+
+		const abstract = words.filter( Boolean ).join( ' ' ).trim();
+		return abstract || null;
+	}
+
+	buildPages( firstPage, lastPage ) {
+		if ( firstPage && lastPage && String( firstPage ) !== String( lastPage ) ) {
+			return `${ firstPage }-${ lastPage }`;
+		}
+		return firstPage || lastPage || null;
+	}
+
+	buildExtra( work ) {
+		const extra = [];
+		if ( work.id ) {
+			extra.push( `OpenAlex: ${ work.id }` );
+		}
+		if ( work.ids && work.ids.pmid ) {
+			extra.push( `PMID: ${ this.stripIdentifierPrefix( work.ids.pmid ) }` );
+		}
+		if ( work.ids && work.ids.pmcid ) {
+			extra.push( `PMCID: ${ this.stripIdentifierPrefix( work.ids.pmcid ) }` );
+		}
+		return extra.join( '\n' );
+	}
+
+	authorToCreator( authorship ) {
+		if ( !authorship || !authorship.author || !authorship.author.display_name ) {
+			return null;
+		}
+
+		const name = authorship.author.display_name.trim();
+		const parts = name.split( /\s+/ );
+
+		if ( parts.length === 1 ) {
+			return {
+				creatorType: 'author',
+				lastName: parts[ 0 ]
+			};
+		}
+
+		return {
+			creatorType: 'author',
+			firstName: parts.slice( 0, -1 ).join( ' ' ),
+			lastName: parts[ parts.length - 1 ]
+		};
+	}
+
+	normalizeDoi( doiUrl ) {
+		if ( !doiUrl || typeof doiUrl !== 'string' ) {
+			return null;
+		}
+		return doiUrl.replace( /^https?:\/\/(?:dx\.)?doi\.org\//i, '' );
+	}
+
+	stripIdentifierPrefix( value ) {
+		if ( !value || typeof value !== 'string' ) {
+			return value;
+		}
+		return value.replace( /^https?:\/\/[^/]+\//i, '' ).replace( /^(?:pmid:|pmcid:)/i, '' );
+	}
+}
+
+module.exports = OpenAlexService;

package/lib/external-apis/PubMedService.js

@@ -0,0 +1,82 @@
+'use strict';
+
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ *
+ * Requests and sanity checks the response from PubMed's API
+ */
+
+const BBPromise = require( 'bluebird' );
+
+class PubMedService {
+
+	/**
+	 * Constructor for PubMed Service object
+	 *
+	 * @param {Object} app   Express app; contains logger, metrics, and configuration
+	 */
+	constructor( app ) {
+		this.mailto = app.conf.mailto;
+	}
+
+	/**
+	 * Requests a PubMed object using any supported identifier
+	 *
+	 * @param  {string}  identifier  Valid PubMed identifier (DOI, PMID, PMCID with PMC prefix,
+	 *                               Manuscript ID, versioned ID)
+	 * @param  {string}  request     original request option
+	 * @return {Object}              BBPromise for converted ID
+	 */
+	convert( identifier, request ) {
+		if ( !identifier ) {
+			return BBPromise.reject( 'No identifier in request' );
+		}
+		const escapedId = encodeURIComponent( identifier );
+		// TODO: Make url configurable
+		const url = 'https://pmc.ncbi.nlm.nih.gov/tools/idconv/api/v1/articles/';
+		const mailto = this.mailto || 'example@example.com'; // set default in case the config lacks it
+
+		const qs = {
+			tool: 'citoid',
+			email: mailto,
+			format: 'json',
+			ids: escapedId
+		};
+
+		let message;
+
+		return request.issueRequest( {
+			uri: url,
+			qs,
+			qsStringifyOptions: { // Prevent email from being URL encoded
+				encode: false
+			}
+		} ).then( ( response ) => {
+
+			const body = response.body;
+			request.logger.log( 'debug/pubmed', `PubMed query made for ids: ${ identifier }` );
+			if ( response.status !== 200 ) {
+				message = `Unexpected HTTP status code: ${ response.statusCode }`;
+				request.logger.log( 'warn/pubmed', message );
+				return BBPromise.reject( message );
+			} else if ( body.status !== 'ok' ) {
+				message = `Unexpected status from PubMed API: ${ body.status }`;
+				request.logger.log( 'warn/pubmed', message );
+				return BBPromise.reject( message );
+			} else if ( body.records.length === 0 ) {
+				message = 'No records from PubMed API';
+				request.logger.log( 'warn/pubmed', message );
+				return BBPromise.reject( message );
+			} else {
+				request.logger.log( 'trace/pubmed', `Successful PubMed query made for ids: ${ identifier }` );
+				return body;
+			}
+		} ).catch( ( error ) => {
+			message = 'Unknown pubmed error';
+			request.logger.log( 'warn/pubmed', message );
+			return BBPromise.reject( message );
+		} );
+	}
+}
+
+module.exports = PubMedService;

package/lib/external-apis/SemanticScholarService.js

@@ -0,0 +1,221 @@
+'use strict';
+
+const BBPromise = require( 'bluebird' );
+
+const { validateZotero } = require( '../Exporter.js' );
+
+const DEFAULT_FIELDS = [
+	'title',
+	'year',
+	'authors',
+	'externalIds',
+	'venue',
+	'publicationVenue',
+	'publicationDate',
+	'abstract',
+	'url',
+	'openAccessPdf'
+];
+
+class SemanticScholarService {
+
+	constructor( app ) {
+		const conf = app.conf.semanticScholar || {};
+		this.api = conf.api || 'https://api.semanticscholar.org/graph/v1';
+		this.apiKey = conf.apiKey || null;
+		this.enabled = !!conf.enabled;
+		this.fields = Array.isArray( conf.fields ) && conf.fields.length ?
+			conf.fields : DEFAULT_FIELDS;
+		this.minIntervalMs = Number.isFinite( conf.minIntervalMs ) ?
+			conf.minIntervalMs : 1000;
+		this.requestTimeout = Number.isFinite( conf.timeout ) ? conf.timeout : 5000;
+		this.searchLimit = Number.isFinite( conf.searchLimit ) ? conf.searchLimit : 2;
+		this.preferForArxiv = conf.preferForArxiv !== false;
+		this._nextRequestAt = 0;
+		this._queue = BBPromise.resolve();
+	}
+
+	issueRequest( request, requestOptions ) {
+		const headers = {};
+		if ( this.apiKey ) {
+			headers[ 'x-api-key' ] = this.apiKey;
+		}
+
+		const options = {
+			uri: `${ this.api }${ requestOptions.path }`,
+			headers,
+			qs: requestOptions.qs,
+			timeout: this.requestTimeout
+		};
+
+		return this.rateLimit( () => request.issueRequest( options ) );
+	}
+
+	rateLimit( callback ) {
+		const scheduled = this._queue.then( () => {
+			const now = Date.now();
+			const waitMs = Math.max( 0, this._nextRequestAt - now );
+			this._nextRequestAt = Math.max( this._nextRequestAt, now ) + this.minIntervalMs;
+			return BBPromise.delay( waitMs ).then( callback );
+		} );
+
+		this._queue = scheduled.reflect();
+		return scheduled;
+	}
+
+	getPaper( paperId, request ) {
+		if ( !paperId ) {
+			return BBPromise.reject( 'No paper id provided' );
+		}
+
+		request.logger.log( 'debug/other', `Making request to Semantic Scholar for paper ${ paperId }` );
+
+		return this.issueRequest( request, {
+			path: `/paper/${ encodeURIComponent( paperId ) }`,
+			qs: {
+				fields: this.fields.join( ',' )
+			}
+		} ).then( ( response ) => {
+			if ( response && response.status === 200 && response.body && response.body.paperId ) {
+				return response.body;
+			}
+			return BBPromise.reject( `No Semantic Scholar paper found for ${ paperId }` );
+		} );
+	}
+
+	search( query, request ) {
+		if ( !query ) {
+			return BBPromise.reject( 'No search query provided' );
+		}
+
+		request.logger.log( 'debug/other', `Making request to Semantic Scholar search for ${ query }` );
+
+		return this.issueRequest( request, {
+			path: '/paper/search',
+			qs: {
+				query,
+				limit: this.searchLimit,
+				fields: this.fields.join( ',' )
+			}
+		} ).then( ( response ) => {
+			if ( response && response.status === 200 && response.body &&
+				Array.isArray( response.body.data ) && response.body.data.length ) {
+				return response.body.data.slice( 0, this.searchLimit );
+			}
+			return BBPromise.reject( `No Semantic Scholar results found for ${ query }` );
+		} );
+	}
+
+	toCitation( paper, originalQuery ) {
+		if ( !paper || !paper.title ) {
+			throw new Error( 'Invalid Semantic Scholar paper payload' );
+		}
+
+		const externalIds = paper.externalIds || {};
+		const doi = externalIds.DOI;
+		const arxiv = externalIds.ArXiv;
+		const abstract = typeof paper.abstract === 'string' ? paper.abstract.trim() : null;
+		const openAccessPdf = paper.openAccessPdf && paper.openAccessPdf.url ?
+			paper.openAccessPdf.url : null;
+		const venue = paper.publicationVenue && paper.publicationVenue.name ?
+			paper.publicationVenue.name : paper.venue;
+		const citation = {
+			itemType: this.resolveItemType( paper ),
+			title: paper.title,
+			url: this.resolveUrl( paper, originalQuery ),
+			libraryCatalog: 'Semantic Scholar',
+			accessDate: ( new Date() ).toISOString().slice( 0, 10 )
+		};
+
+		if ( doi ) {
+			citation.DOI = doi;
+		}
+		if ( abstract ) {
+			citation.abstractNote = abstract;
+		}
+		if ( venue ) {
+			citation.publicationTitle = venue;
+		}
+		if ( paper.publicationDate ) {
+			citation.date = paper.publicationDate;
+		} else if ( paper.year ) {
+			citation.date = String( paper.year );
+		}
+		if ( openAccessPdf && citation.url && !citation.url.startsWith( 'https://doi.org/' ) &&
+			!citation.url.startsWith( 'https://arxiv.org/abs/' ) ) {
+			citation.url = openAccessPdf;
+		}
+		if ( paper.paperId || arxiv ) {
+			const extra = [];
+			if ( paper.paperId ) {
+				extra.push( `Semantic Scholar Paper ID: ${ paper.paperId }` );
+			}
+			if ( arxiv ) {
+				extra.push( `arXiv: ${ arxiv }` );
+			}
+			citation.extra = extra.join( '\n' );
+		}
+		if ( Array.isArray( paper.authors ) && paper.authors.length ) {
+			citation.creators = paper.authors.map( ( author ) => this.authorToCreator( author ) )
+				.filter( Boolean );
+		}
+
+		return validateZotero( null, citation );
+	}
+
+	resolveItemType( paper ) {
+		const venueType = paper.publicationVenue && paper.publicationVenue.type ?
+			paper.publicationVenue.type.toLowerCase() : '';
+		const externalIds = paper.externalIds || {};
+
+		if ( venueType === 'conference' ) {
+			return 'conferencePaper';
+		}
+		if ( venueType === 'journal' ) {
+			return 'journalArticle';
+		}
+		if ( externalIds.ArXiv && !venueType ) {
+			return 'preprint';
+		}
+		return 'journalArticle';
+	}
+
+	resolveUrl( paper, originalQuery ) {
+		const externalIds = paper.externalIds || {};
+		if ( externalIds.DOI ) {
+			return `https://doi.org/${ externalIds.DOI }`;
+		}
+		if ( externalIds.ArXiv ) {
+			return `https://arxiv.org/abs/${ externalIds.ArXiv }`;
+		}
+		if ( typeof originalQuery === 'string' && originalQuery.startsWith( 'http' ) ) {
+			return originalQuery;
+		}
+		if ( paper.openAccessPdf && paper.openAccessPdf.url ) {
+			return paper.openAccessPdf.url;
+		}
+		return paper.url;
+	}
+
+	authorToCreator( author ) {
+		if ( !author || !author.name ) {
+			return null;
+		}
+
+		const parts = author.name.trim().split( /\s+/ );
+		if ( parts.length === 1 ) {
+			return {
+				creatorType: 'author',
+				lastName: parts[ 0 ]
+			};
+		}
+
+		return {
+			creatorType: 'author',
+			firstName: parts.slice( 0, -1 ).join( ' ' ),
+			lastName: parts.at( -1 )
+		};
+	}
+}
+
+module.exports = SemanticScholarService;

package/lib/external-apis/WaybackMachine.js

@@ -0,0 +1,92 @@
+'use strict';
+
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ *
+ * Supplies methods to send requests to the Wayback Machine APIs
+ */
+
+/* Import Modules */
+const BBPromise = require( 'bluebird' );
+
+/**
+ * Constructor for CitoidService object
+ *
+ * @param {Object} app   Express app; contains logger, metrics, and configuration
+ */
+class WaybackMachine {
+
+	/**
+	 * Constructor for CitoidService object
+	 *
+	 * @param {Object} app   Express app; contains logger, metrics, and configuration
+	 */
+	constructor( app ) {}
+
+	/**
+	 * Replaces timestamp (i.e. 20210618033109) with ISO date (i.e. 2021-06-18)
+	 *
+	 * @param  {Object} snapshot Metadata about archived url
+	 * @return {Object}          Mutated snapshot object with fixed date
+	 */
+
+	cleanResults( snapshot ) {
+		const year = snapshot.timestamp.slice( 0, 4 );
+		const month = snapshot.timestamp.slice( 4, 6 );
+		const day = snapshot.timestamp.slice( 6, 8 );
+		snapshot.timestamp = `${ year }-${ month }-${ day }`;
+		return snapshot;
+	}
+
+	/**
+	 * Requests to Wayback Availability JSON API at https://archive.org/help/wayback_api.php
+	 *
+	 * @param  {string} url            requested url
+	 * @param  {Object} request        request object for entire citoid request - has utilities
+	 * @return {Object}                metadata about most recent snapshot
+	 */
+	availability( url, request ) {
+
+		if ( !url ) {
+			return BBPromise.reject( 'No url in request' );
+		}
+
+		if ( !request ) {
+			return BBPromise.reject( 'Missing request argument' );
+		}
+
+		request.logger.log( 'debug/other', 'Making request to Wayback Machine availability API' );
+
+		const endpoint = 'http://archive.org/wayback/available';
+
+		const qs = { // Basic query parameters
+			url: url // Encodes url
+		};
+
+		return request.issueRequest(
+			{
+				uri: endpoint,
+				headers: {
+					'User-Agent': this.userAgent
+				},
+				qs: qs
+			} )
+			.then( ( res ) => {
+				if ( res && res.status === 200 && res.body.archived_snapshots && res.body.archived_snapshots.closest && res.body.archived_snapshots.closest.status === '200' ) {
+					request.logger.log( 'debug/wayback', 'Archived copy of ' + url + ' available' );
+					return this.cleanResults( res.body.archived_snapshots.closest );
+				} else {
+					request.logger.log( 'debug/wayback', 'No acceptable results from WaybackMachine availability service for url ' + url );
+					return BBPromise.reject( 'No acceptable results from WaybackMachine availability service' );
+				}
+			} ).catch( ( e ) => {
+				request.logger.log( 'debug/wayback', e );
+				return BBPromise.reject( 'WaybackMachine availability service unavailable' );
+			} );
+
+	}
+
+}
+
+/* Exports */
+module.exports = WaybackMachine;