@florajs/datasource-solr 3.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-present BörseGo AG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,11 @@
+# @florajs/datasource-solr
+
+![](https://github.com/florajs/datasource-solr/workflows/ci/badge.svg)
+[![NPM version](https://img.shields.io/npm/v/@florajs/datasource-solr.svg?style=flat)](https://www.npmjs.com/package/@florajs/datasource-solr)
+[![NPM downloads](https://img.shields.io/npm/dm/@florajs/datasource-solr.svg?style=flat)](https://www.npmjs.com/package/@florajs/datasource-solr)
+
+Solr connection for [Flora](https://github.com/florajs/flora).
+
+## License
+
+[MIT](LICENSE)

package/index.js

@@ -0,0 +1,367 @@
+'use strict';
+
+const http = require('http');
+const querystring = require('querystring');
+
+const _ = require('lodash');
+const { ImplementationError } = require('@florajs/errors');
+
+const SUPPORTED_FILTERS = ['equal', 'notEqual', 'lessOrEqual', 'greaterOrEqual', 'range'];
+
+const NO_LIMIT = 1000000;
+
+/**
+ * Return true if two filters on same attribute can be used as single range filter.
+ *
+ * @param {Array.<Object>} filters
+ * @return {boolean}
+ * @private
+ */
+function filterRangeQueries(filters) {
+    if (filters.length !== 2) return false;
+
+    const rangeOperators = ['lessOrEqual', 'greaterOrEqual'];
+    const operator1 = filters[0].operator;
+    const operator2 = filters[1].operator;
+
+    return rangeOperators.includes(operator1) && rangeOperators.includes(operator2);
+}
+
+/**
+ * Create range filter from a greatOrEqual and lessOrEqual filter.
+ *
+ * @param {Array.<string>} attributeFilters
+ * @return {Object}
+ * @private
+ */
+function createRangeFilter(attributeFilters) {
+    const rangeFilter = { attribute: attributeFilters[0].attribute, operator: 'range' };
+
+    // make sure greaterOrEqual filter comes first
+    attributeFilters.sort((filter) => (filter.operator === 'greaterOrEqual' ? -1 : 1));
+
+    rangeFilter.value = [attributeFilters[0].value, attributeFilters[1].value];
+    return rangeFilter;
+}
+
+/**
+ * Convert greaterOrEqual and lessOrEqual filters on same attribute
+ * to a single range filter.
+ *
+ * @param {Array.<Object>} filters
+ * @return {Array.<Object>}
+ * @private
+ */
+function rangify(filters) {
+    const hasRangeableFilters = filters.some((filter) => ['lessOrEqual', 'greaterOrEqual'].includes(filter.operator));
+    if (!hasRangeableFilters) {
+        return filters;
+    }
+
+    const groupedAttrs = _.groupBy(filters, 'attribute');
+    const rangeQueries = _.filter(groupedAttrs, filterRangeQueries);
+
+    if (!rangeQueries.length) return filters;
+
+    const rangeQueryAttrs = rangeQueries.map((rangeQuery) => rangeQuery[0].attribute);
+
+    // copy non-range query attributes
+    return filters
+        .filter((filter) => !rangeQueryAttrs.includes(filter.attribute))
+        .concat(rangeQueries.map(createRangeFilter));
+}
+
+/**
+ * @param {string} value
+ * @return {string}
+ * @private
+ */
+function escapeSpecialChars(value) {
+    const reservedKeywordRegex = /\b(AND|NOT|OR)\b/g;
+
+    if (reservedKeywordRegex.test(value)) {
+        value = value.replace(reservedKeywordRegex, (keyword) => keyword.toLowerCase());
+    }
+
+    return value.replace(/([\\/+\-&|!(){}[\]^"~*?:])/g, '\\$1');
+}
+
+/**
+ * @param {string} value
+ * @return {string}
+ * @private
+ */
+function escapeSpecialCharsSolrSyntax(value) {
+    return value.replace(/([\\/+\-&|!(){}[\]^~?])/g, '\\$1');
+}
+
+/**
+ * Escape strings and convert boolean values.
+ *
+ * @param {*} value
+ * @param {boolean} exposeSolrSyntax
+ * @return {*}
+ * @private
+ */
+function escapeValueForSolr(value, exposeSolrSyntax) {
+    if (typeof value === 'string') {
+        value = exposeSolrSyntax ? escapeSpecialCharsSolrSyntax(value) : escapeSpecialChars(value);
+    } else if (typeof value === 'boolean') value = value === false ? 0 : 1;
+    return value;
+}
+
+/**
+ * @param {Object} filter
+ * @return {string}
+ * @private
+ */
+function convertFilterToSolrSyntax(filter) {
+    const { operator } = filter;
+    let { value } = filter;
+
+    if (!SUPPORTED_FILTERS.includes(filter.operator)) {
+        throw new ImplementationError(`DataSource "solr" does not support "${operator}" filters`);
+    }
+
+    if (!Array.isArray(filter.attribute)) {
+        value = escapeValueForSolr(value);
+        if (Array.isArray(value)) {
+            if (operator === 'range') value = '[' + value[0] + ' TO ' + value[1] + ']';
+            else value = '(' + value.join(' OR ') + ')';
+        }
+
+        if (operator !== 'notEqual') {
+            if (operator === 'greaterOrEqual') value = '[' + value + ' TO *]';
+            if (operator === 'lessOrEqual') value = '[* TO ' + value + ']';
+            return filter.attribute + ':' + value;
+        }
+        return '-' + filter.attribute + ':' + value;
+    }
+
+    // convert composite keys to SOLR syntax
+    return value
+        .map((values) => {
+            const conditions = values.map((val, index) => filter.attribute[index] + ':' + escapeValueForSolr(val));
+            return '(' + conditions.join(' AND ') + ')';
+        })
+        .join(' OR ');
+}
+
+function buildSolrFilterString(floraFilters) {
+    const orConditions = floraFilters.map((andFilters) => {
+        if (andFilters.length > 1) andFilters = rangify(andFilters);
+        const conditions = andFilters.map(convertFilterToSolrSyntax);
+        return '(' + conditions.join(' AND ') + ')';
+    });
+
+    if (orConditions.length > 1) return '(' + orConditions.join(' OR ') + ')';
+
+    return orConditions.join('');
+}
+
+/**
+ * @param {Array.<Object>} floraOrders
+ * @return {string}
+ * @private
+ */
+function buildSolrOrderString(floraOrders) {
+    return floraOrders.map((order) => order.attribute + ' ' + order.direction).join(',');
+}
+
+function parseData(str) {
+    try {
+        return JSON.parse(str);
+    } catch (e) {
+        return new Error('Could not parse response: ' + str);
+    }
+}
+
+function prepareQueryAddition(queryAdditions) {
+    return queryAdditions
+        .replace(/[\r\n]+/g, ' ')
+        .replace(/\s{2,}/g, ' ')
+        .trim();
+}
+
+/**
+ * @param {Object} servers
+ * @return {Object}
+ */
+function getUrlGenerators(servers) {
+    return Object.keys(servers).reduce((acc, server) => {
+        const { urls } = servers[server];
+
+        acc[server] = (function* urlGenerator() {
+            while (true) {
+                for (let i = 0, l = urls.length; i < l; ++i) {
+                    yield urls[i];
+                }
+            }
+        })();
+
+        return acc;
+    }, {});
+}
+
+/**
+ *
+ * @param {string} requestUrl
+ * @param {Object} params
+ * @param {Object} requestOptions
+ * @param {Agent} agent
+ * @returns {Promise}
+ * @private
+ */
+function querySolr(requestUrl, params, requestOptions, agent) {
+    return new Promise((resolve, reject) => {
+        const options = {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+            timeout: requestOptions.connectTimeout,
+            agent
+        };
+
+        const req = http.request(requestUrl, options, (res) => {
+            const chunks = [];
+
+            res.on('data', (chunk) => chunks.push(chunk));
+
+            res.on('end', () => {
+                const data = parseData(Buffer.concat(chunks).toString('utf8'));
+
+                if (res.statusCode >= 400 || data instanceof Error) {
+                    const error = new Error(`Solr error: ${res.statusCode} ${http.STATUS_CODES[res.statusCode]}`);
+                    return reject(error);
+                }
+
+                return resolve({ totalCount: data.response.numFound, data: data.response.docs });
+            });
+        });
+
+        req.setTimeout(requestOptions.requestTimeout, () => req.abort());
+
+        req.write(querystring.stringify(params)); // add params to POST body
+
+        req.on('error', (err) => {
+            err.message = `Solr error: ${err.message} (${options.host})`;
+            reject(err);
+        });
+
+        req.end();
+    });
+}
+
+function prepareSearchTerm(request) {
+    const escapedSearchTerm = escapeValueForSolr(request.search, request.exposeSolrSyntax);
+    const allowedSearchFields =
+        request.allowedSearchFields && request.allowedSearchFields.trim()
+            ? request.allowedSearchFields.trim().split(',')
+            : [];
+
+    if (!allowedSearchFields.length) {
+        return escapedSearchTerm;
+    }
+
+    const regex = new RegExp(`^(?<field>${allowedSearchFields.join('|')}):(?<search>.+)`);
+    const match = regex.exec(request.search);
+
+    if (match === null) {
+        return escapedSearchTerm;
+    }
+
+    return `(${match.groups.field}:"${match.groups.search}")`;
+}
+
+class DataSource {
+    /**
+     * @param {Api} api
+     * @param {Object} config
+     */
+    constructor(api, config) {
+        this.options = config;
+        this._urls = getUrlGenerators(config.servers);
+        this._status = config._status;
+        delete config._status;
+
+        this._agent = new http.Agent({
+            maxSockets: 5,
+            keepAlive: true,
+            keepAliveMsecs: 10000
+        });
+    }
+
+    /**
+     * @public
+     */
+    prepare() {}
+
+    /**
+     * @param {Object} request
+     * @returns {Promise<Object>}
+     */
+    async process(request) {
+        const server = request.server || 'default';
+        const queryParts = [];
+        const params = { wt: 'json' };
+        const serverOpts = this.options.servers;
+
+        if (!serverOpts[server]) throw new Error(`Server "${server}" not defined`);
+
+        const requestUrl = this._urls[server].next().value + request.collection + '/select';
+
+        if (request.attributes) params.fl = request.attributes.join(',');
+        if (request.order) params.sort = buildSolrOrderString(request.order);
+
+        if (request.df) params.df = request.df;
+        if (request.search && request.search.trim()) queryParts.push(prepareSearchTerm(request));
+        if (request.filter) {
+            queryParts.push(buildSolrFilterString(request.filter));
+        }
+        if (request.queryAddition) queryParts.push(prepareQueryAddition(request.queryAddition));
+        if (queryParts.length === 0) queryParts.push('*:*');
+
+        // overwrite SOLR default limit for sub-resource processing
+        if (!request.limit) request.limit = NO_LIMIT;
+        if (request.page) params.start = (request.page - 1) * request.limit;
+
+        if (!request.limitPer) params.rows = request.limit;
+        else {
+            Object.assign(params, {
+                group: 'true',
+                'group.format': 'simple',
+                'group.main': 'true',
+                'group.field': request.limitPer,
+                'group.limit': request.limit,
+                rows: NO_LIMIT // disable default limit because groups are returned as list
+            });
+        }
+
+        params.q = queryParts.join(' AND ');
+
+        if (request._explain) Object.assign(request._explain, { url: requestUrl, params });
+        if (this._status) this._status.increment('dataSourceQueries');
+
+        const requestOpts = {
+            connectTimeout: serverOpts[server].connectTimeout || 2000,
+            requestTimeout: serverOpts[server].requestTimeout || 5000
+        };
+
+        return querySolr(requestUrl, params, requestOpts, this._agent);
+    }
+
+    /**
+     * @returns {Promise}
+     */
+    close() {
+        return Promise.resolve();
+    }
+
+    /**
+     * @param {...string} args
+     */
+    escape(...args) {
+        return escapeValueForSolr(...args);
+    }
+}
+
+module.exports = DataSource;

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@florajs/datasource-solr",
+  "version": "3.0.0",
+  "description": "Solr connection for Flora",
+  "main": "index.js",
+  "scripts": {
+    "test": "eslint . && mocha",
+    "lint": "eslint ."
+  },
+  "files": [
+    "index.js"
+  ],
+  "pre-commit": "test",
+  "author": {
+    "name": "BörseGo AG",
+    "email": "it@boerse-go.de"
+  },
+  "keywords": [
+    "florajs",
+    "solr"
+  ],
+  "homepage": "https://github.com/florajs/datasource-solr",
+  "license": "MIT",
+  "repository": "florajs/datasource-solr",
+  "contributors": [
+    {
+      "name": "Marco Baumgartl",
+      "email": "marco.baumgartl@boerse-go.de"
+    },
+    {
+      "name": "Nico Kaiser",
+      "email": "nico.kaiser@boerse-go.de"
+    },
+    {
+      "name": "Mark Plomer",
+      "email": "mark.plomer@boerse-go.de"
+    }
+  ],
+  "engines": {
+    "node": ">=12"
+  },
+  "dependencies": {
+    "@florajs/errors": "^3.0.0",
+    "lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.0",
+    "@types/mocha": "^9.0.0",
+    "@types/nock": "^11.1.0",
+    "chai": "^4.3.4",
+    "eslint": "^8.6.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "mocha": "^9.1.3",
+    "nock": "^13.2.1",
+    "pre-commit": "^1.2.2",
+    "prettier": "^2.5.1"
+  }
+}