citeclaw 2.0.0

package/config.dev.yaml

@@ -0,0 +1,124 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: 0
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 250
+
+# Logger info
+logging:
+  level: trace
+  streams:
+    - type: stdout
+
+# Metrics reporters
+# eslint-disable-next-line yml/no-empty-mapping-value
+metrics:
+#  - type: prometheus
+#    port: 9100
+#  - type: statsd
+#    host: localhost
+#    port: 8125
+
+services:
+  - name: citoid
+    # a relative path or the name of an npm package, if different from name
+    module: ./app.js
+    # optionally, a version constraint of the npm package
+    # version: ^0.4.0
+    # per-service config
+    conf:
+      # the port to bind to
+      port: 1970
+      # IP address to bind to, all IPs by default
+      # interface: localhost # uncomment to only listen on localhost
+      # more per-service config settings
+      # the location of the spec, defaults to spec.yaml if not specified
+      # spec: ./spec.template.yaml
+      # allow cross-domain requests to the API (default '*')
+      cors: "*"
+      # to disable use:
+      # cors: false
+      # to restrict to a particular domain, use:
+      # cors: restricted.domain.org
+      # content for the CSP headers
+      # csp: false  # uncomment this line to disable sending them
+      # URL of the outbound proxy to use (complete with protocol)
+      # proxy: http://my.proxy.org:8080
+      # the list of domains for which not to use the proxy defined above
+      # no_proxy_list:
+      #   - domain1.com
+      #   - domain2.org
+      # the list of incoming request headers that can be logged; if left empty,
+      # the following headers are allowed: cache-control, content-length,
+      # content-type, if-match, user-agent, x-request-id
+      # log_header_whitelist:
+      #   - cache-control
+      #   - content-length
+      #   - content-type
+      #   - if-match
+      #   - user-agent
+      #   - x-request-id
+      # enable use of wayback machine from archive.org if initial url request fails
+      wayback: false
+      # enable use of NIH PubMed id converter to add pmids and pmcids to citations
+      # at the end of a request; defaults to true if undefined
+      pubmed: false
+      # enable Semantic Scholar as a supplementary source for DOI/arXiv/search queries
+      semanticScholar:
+        enabled: false
+        api: https://api.semanticscholar.org/graph/v1
+        apiKey: false
+        minIntervalMs: 1000
+        timeout: 5000
+        searchLimit: 2
+        preferForArxiv: true
+      # enable OpenAlex as a supplementary source for DOI/PMID/PMCID/search queries
+      openAlex:
+        enabled: false
+        api: https://api.openalex.org
+        apiKey: false
+        timeout: 5000
+        searchLimit: 2
+      # enable use of zotero's translation-server service to query for URLs;
+      # defaults to true if undefined
+      zotero: true
+      # URL where to contact Zotero
+      zoteroInterface: 127.0.0.1
+      # zotero's server port
+      zoteroPort: 1969
+      # whether the proxy should be used to contact zotero
+      zoteroUseProxy: false
+      allowPrivateAddresses: false
+      maxRedirects: 5
+      # the user agent to use when issuing requests
+      user_agent: citoid-test-env
+      # Contact e-mail address. Used for making CrossRef and PubMed requests
+      mailto: example@example.com
+      # Force certain requests to return with partial data using this timeout in milliseconds
+      timeout: false
+      outgoing_request_throttle:
+        enabled: true
+        delayMs: 3100
+        skipHosts:
+          - localhost
+          - 127.0.0.1
+          - ::1
+      # Max JSON POST body size limit.
+      # max_body_size: 100kb
+      # the template used for contacting the MW API
+      mwapi_req:
+        method: post
+        uri: https://{{domain}}/w/api.php
+        headers:
+          user-agent: "{{user-agent}}"
+        body: "{{ default(request.query, {}) }}"
+      # the template used for contacting RESTBase
+      restbase_req:
+        method: "{{request.method}}"
+        uri: https://{{domain}}/api/rest_v1/{+path}
+        query: "{{ default(request.query, {}) }}"
+        headers: "{{request.headers}}"
+        body: "{{request.body}}"

package/config.prod.yaml

@@ -0,0 +1,123 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 500
+
+# Logger info
+logging:
+  level: warn
+  streams:
+    # Use syslog -> logstash
+    - type: syslog
+      host: localhost
+      port: 10514
+      prefix: "@cee: "
+      name: node
+
+# Metrics reporters
+metrics:
+  - type: prometheus
+    port: 9102
+#  - type: statsd
+#    host: localhost
+#    port: 8125
+
+services:
+  - name: citoid
+    # a relative path or the name of an npm package, if different from name
+    module: ./app.js
+    # optionally, a version constraint of the npm package
+    # version: ^0.4.0
+    # per-service config
+    conf:
+      port: 1970
+      # interface: localhost # uncomment to only listen on localhost
+      # more per-service config settings
+      # the location of the spec, defaults to spec.yaml if not specified
+      # spec: ./spec.template.yaml
+      # allow cross-domain requests to the API (default '*')
+      cors: "*"
+      # to disable use:
+      # cors: false
+      # to restrict to a particular domain, use:
+      # cors: restricted.domain.org
+      # content for the CSP headers
+      # csp: false  # uncomment this line to disable sending them
+      # URL of the outbound proxy to use (complete with protocol)
+      # proxy: http://my.proxy.org:8080
+      # the list of domains for which not to use the proxy defined above
+      # no_proxy_list:
+      #   - domain1.com
+      #   - domain2.org
+      # the list of incoming request headers that can be logged; if left empty,
+      # the following headers are allowed: cache-control, content-length,
+      # content-type, if-match, user-agent, x-request-id
+      # log_header_whitelist:
+      #   - cache-control
+      #   - content-length
+      #   - content-type
+      #   - if-match
+      #   - user-agent
+      #   - x-request-id
+      # the user agent to use when issuing requests
+      # user_agent: service-template-node
+      # the template used for contacting the MW API
+      # enable use of wayback machine from archive.org if initial url request fails
+      wayback: true
+      # enable use of NIH PubMed id converter to add pmids and pmcids to citations
+      # at the end of a request; defaults to true if undefined
+      pubmed: false
+      # enable Semantic Scholar as a supplementary source for DOI/arXiv/search queries
+      semanticScholar:
+        enabled: false
+        api: https://api.semanticscholar.org/graph/v1
+        apiKey: false
+        minIntervalMs: 1000
+        timeout: 5000
+        searchLimit: 2
+        preferForArxiv: true
+      # enable OpenAlex as a supplementary source for DOI/PMID/PMCID/search queries
+      openAlex:
+        enabled: false
+        api: https://api.openalex.org
+        apiKey: false
+        timeout: 5000
+        searchLimit: 2
+      # enable use of zotero's translation-server service to query for URLs;
+      # defaults to true if undefined
+      zotero: true
+      # URL where to contact Zotero
+      zoteroInterface: 127.0.0.1
+      # zotero's server port
+      zoteroPort: 1969
+      # whether the proxy should be used to contact zotero
+      zoteroUseProxy: false
+      # Contact e-mail address. Used for making CrossRef and PubMed requests
+      mailto: example@example.com
+      # Force certain requests to return with partial data using this timeout in milliseconds
+      timeout: 5000
+      outgoing_request_throttle:
+        enabled: true
+        delayMs: 3100
+        skipHosts:
+          - localhost
+          - 127.0.0.1
+          - ::1
+      mwapi_req:
+        method: post
+        uri: https://mw-api-int-ro.discovery.wmnet:4446/w/api.php
+        headers:
+          host: "{{request.params.domain}}"
+          user-agent: "{{user-agent}}"
+        body: "{{ default(request.query, {}) }}"
+      # the template used for contacting RESTBase
+      restbase_req:
+        method: "{{request.method}}"
+        uri: http://restbase.svc.eqiad.wmnet/{{domain}}/v1/{+path}
+        query: "{{ default(request.query, {}) }}"
+        headers: "{{request.headers}}"
+        body: "{{request.body}}"

package/lib/Citation.js

@@ -0,0 +1,101 @@
+'use strict';
+
+/**
+ * An object representing a single citation
+ */
+
+class Citation {
+
+	/**
+	 * Constructor for Citation object
+	 *
+	 * @param {string} [idType] Type of citation ID
+	 * @param {string} [idValue] Citation ID value
+	 */
+	constructor( idType, idValue ) {
+		// Valid id types from which to create private variables, setters, and getter
+		const validIdTypes = [ 'doi', 'isbn', 'oclc', 'pmcid', 'pmid', 'qid', 'any', 'url' ];
+
+		// Initialize all possible ID types to null and create getters/setters dynamically
+		validIdTypes.forEach( ( type ) => {
+			// Initialize private property
+			Object.defineProperty( this, `_${ type }`, {
+				value: null,
+				writable: true,
+				enumerable: false,
+				configurable: true
+			} );
+
+			// Create getters and setters for each id type
+			Object.defineProperty( this, type, {
+				get() {
+					return this[ `_${ type }` ];
+				},
+				// Converts falsey values to null and all others to string
+				set( value ) {
+					if ( typeof value === 'boolean' ) { // This should never happen, but just in case
+						value = value ? '' : null;
+					}
+					if ( value === undefined ) {
+						value = null;
+					}
+					this[ `_${ type }` ] = ( value === null ) ? value : String( value );
+				},
+				enumerable: true,
+				configurable: true
+			} );
+		} );
+
+		// Initialize base properties
+		this.idType = null;
+		this.idValue = null;
+
+		// Throw errors if idType is invalid, but fail gracefully for idValue
+		if ( idType && idValue ) {
+			if ( typeof idType !== 'string' ) {
+				throw new Error( 'idType must be a string' );
+			}
+			if ( !validIdTypes.includes( idType ) ) {
+				throw new Error( `Invalid idType: ${ idType }. Must be one of: ${ validIdTypes.join( ', ' ) }` );
+			}
+
+			this.idType = idType;
+
+			if ( typeof idValue === 'boolean' ) {
+				this.idValue = ''; // Mimic set() behaviour above; guaranteed to be true if this block is reached
+			}
+
+			this.idValue = String( idValue ); // Mimic set() behaviour above
+			this[ idType ] = idValue; // Uses set() func for the given property initialised above
+		}
+
+		// For resolved urls different from original url, i.e. redirects followed
+		this.resolvedUrl = null;
+		// HTTP response object for head query to resolved url
+		this.response = null;
+
+		// Wayback machine parameters
+		this.archiveUrl = null;
+		this.archiveDate = null;
+
+		// Whether or not PubMed data has been requested yet.
+		// Either false or a Promise
+		this.hasRequestedPubMed = false;
+
+		this.content = {}; // Initialise empty object for fields
+		this.format = null; // Target format
+		this.formattedContent = {}; // Content translated into targeted format
+		this.source = []; // Array of sources for the content
+
+		this.error = null; // CitoidError; also contains responseCode
+	}
+
+	setWaybackParams( waybackResults ) {
+		if ( waybackResults.status === '200' ) {
+			this.archiveUrl = waybackResults.url;
+			this.archiveDate = waybackResults.timestamp;
+		}
+	}
+}
+
+module.exports = Citation;

package/lib/CitoidError.js

@@ -0,0 +1,61 @@
+'use strict';
+
+/**
+ * An object representing a citoid error response
+ */
+
+/* Import Local Modules */
+const AddressError = require( './hostIsAllowed' ).AddressError;
+
+class CitoidError {
+
+	/**
+	 * Construct with optional parameters
+	 *
+	 * @param  {Error} error       Optional Error if available
+	 * @param  {string} message    Message to go into response
+	 * @param  {number} responseCode  Response code for citoid to reply
+	 * @param  {string} contentType   Optional content type if error is 415
+	 * @return {Object}
+	 */
+	constructor( error, message, responseCode, contentType ) {
+		this.error = error;
+		this.message = message;
+		this.responseCode = responseCode;
+		this.contentType = contentType;
+		this.build();
+	}
+
+	build() {
+		if ( this.error && this.error.name === 'HTTPError' && this.error.body && this.error.body.internalURI ) {
+			this.message = `Unable to load URL ${ this.error.body.internalURI }`;
+			this.responseCode = 404;
+		} else if ( this.error && this.error instanceof AddressError ) {
+			this.message = this.error.message;
+			this.responseCode = 400;
+		}
+		if ( this.responseCode === 415 && !this.message ) {
+			this.message = 'The remote document is not in a supported format';
+		}
+		if ( !this.responseCode ) {
+			this.responseCode = 500;
+		}
+		if ( !this.message ) {
+			this.message = 'Unknown error';
+		}
+	}
+
+	// Get what will go into the body of the response
+	getBody() {
+		if ( this.contentType ) {
+			return {
+				error: this.message,
+				contentType: this.contentType
+			};
+		} else {
+			return { error: this.message };
+		}
+	}
+}
+
+module.exports = CitoidError;

package/lib/CitoidRequest.js

@@ -0,0 +1,150 @@
+'use strict';
+
+/**
+ * An object corresponding to each request to the citoid service
+ */
+
+const BBPromise = require( 'bluebird' );
+const { CookieJar } = require( 'tough-cookie' );
+
+const CitoidResponse = require( './CitoidResponse.js' );
+
+class CitoidRequest {
+
+	/**
+	 * Constructor for CitoidRequest object
+	 *
+	 * @param {Object} req          raw request object from express
+	 * @param {Object} app
+	 */
+	constructor( req, app ) {
+
+		this.conf = app.conf;
+		this.stats = app.metrics;
+
+		this.request = req;
+		this.logger = this.request.logger;
+		this.format = this.request.params.format[ 0 ];
+		this.search = this.request.params.search[ 0 ].trim();
+
+		this.response = new CitoidResponse(); // Prepare an empty response
+		// Array of Promises for cr with Citation objects added in
+		// response.citations from different kinds of queries (i.e. from PMC or PMID)
+		this.getResults = [];
+
+		// Possible extra ids - not guaranteed to be set
+		this.url = null;
+		this.doi = null;
+		this.oclc = null;
+
+		// Cookie jar
+		this.jar = new CookieJar();
+
+		this.exporter = app.citoid.exporter;
+
+		this.build();
+
+	}
+
+	/**
+	 * Modify variables from request object
+	 */
+	build() {
+
+		this.acceptLanguage = this.request.headers[ 'accept-language' ];
+		this.format = encodeURIComponent( this.format );
+
+		const aua = this.request.headers[ 'api-user-agent' ];
+		if ( aua && /^MediaWiki-JS/i.test( aua ) ) {
+			this.apiUserAgent = 'mediawikijs';
+		} else {
+			this.apiUserAgent = 'other';
+		}
+
+	}
+
+	fillBody() {
+		let success = false; // Bool if no citations successfully complete
+		const exportPromises = []; // Promises for exported citations
+
+		// Prevent body from accidentally being overwritten
+		if ( this.response.body ) {
+			return BBPromise.reject( 'Body already filled' );
+		}
+
+		if ( !this.exporter ) {
+			return BBPromise.reject( 'No exporter registered with citoid service.' );
+		}
+
+		// Errors that occur before citations are created, i.e. issues with requests
+		if ( !this.response.citations ||
+			!Array.isArray( this.response.citations ) ||
+			this.response.citations.length < 1
+		) {
+			if ( !this.response.error && this.request.outgoingRequestError ) {
+				this.response.error = this.request.outgoingRequestError;
+			}
+
+			this.response.fillError();
+			return BBPromise.resolve( this );
+		}
+
+		// Create export Promises for each Citation in citations Array
+		const addCitation = ( citation ) => {
+			// Add Citations' exported content to response body
+			if ( !citation.error ) {
+				// Assumes citation.format is set- must be set in CitoidService requestTo methods
+				if ( citation.format === 'bibtex' ) {
+					this.response.body = ( this.response.body ? this.response.body : '' ) +
+                        citation.formattedContent;
+				} else {
+					// Create Array if not already an Array
+					this.response.body = this.response.body ? this.response.body : [];
+					this.response.body.push( citation.formattedContent );
+				}
+				success = true;
+			}
+			return this;
+		};
+
+		for ( let i = 0; i < this.response.citations.length; i++ ) {
+			if ( !this.response.citations[ i ].error ) {
+				exportPromises.push(
+					this.exporter.export(
+						this.response.citations[ i ], this.request
+					).then( addCitation )
+				);
+			}
+		}
+
+		// Wait for all export Promises to complete, then either fill with error or return
+		// cr containing successful citations
+		return BBPromise.all( exportPromises.map( ( x ) => x.reflect() ) ).then( () => {
+			if ( success ) {
+				this.response.responseCode = 200;
+			} else {
+				this.response.fillError();
+			}
+
+			// Metrics for success ratio SLO
+			this.stats.apiuseragent.increment( 1, [
+				this.apiUserAgent,
+				this.response.responseCode
+			] );
+
+			if ( this.response.responseCode === 500 ) {
+				this.logger.log( 'warn/citoidRequest', { msg: '500 internal server error: ', cr: this } );
+			} else {
+				this.logger.log( 'debug/citoidRequest', { msg: 'Full response complete, sending: ', cr: this } );
+			}
+			return BBPromise.resolve( this );
+		} );
+	}
+
+	getResponse() {
+		throw new Error( 'Method getResponse should be overridden' );
+	}
+
+}
+
+module.exports = CitoidRequest;

package/lib/CitoidResponse.js

@@ -0,0 +1,44 @@
+'use strict';
+
+/**
+ * An object corresponding to each request to the citoid service
+ */
+
+const CitoidError = require( './CitoidError.js' );
+
+class CitoidResponse {
+
+	/**
+	 * Constructor for CitoidRequest object
+	 */
+	constructor() {
+
+		this.citations = []; // Array of Citation objects
+		// Array of sources of the metadata, i.e. CrossRef, Zotero, citoid, etc.
+		this.source = [];
+
+		this.responseCode = null; // response code to send
+		this.error = null; // CitoidError for entire response (each Citation also may have one)
+		this.body = null; // Body constructed from Citation objects
+	}
+
+	fillError() {
+		// Arbitrarily pick first error available if one isn't already present
+		if ( !this.error && this.citations && this.citations.length > 0 ) {
+			for ( const c in this.citations ) {
+				if ( this.citations[ c ].error ) {
+					this.error = this.citations[ c ].error;
+					break;
+				}
+			}
+		}
+		// Fallback blank error if it hasn't been set elsewhere in the callstack
+		if ( !this.error ) {
+			this.error = new CitoidError();
+		}
+		this.responseCode = this.error.responseCode;
+		this.body = this.error.getBody();
+	}
+}
+
+module.exports = CitoidResponse;