@tiledesk/tiledesk-server 2.25.0 → 2.26.0

package/CHANGELOG.md

@@ -5,6 +5,9 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.26.0
+- Added pubmodules proxy route for docsparse
+
 # 2.25.0
 - Added support for OpenRouter
 - Added support for Gemini Agent Platform

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.25.0",
+  "version": "2.26.0",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",

package/pubmodules/docsparse/docsparseRoute.js

@@ -0,0 +1,94 @@
+var express = require('express');
+var router = express.Router({ mergeParams: true });
+var axios = require('axios');
+var querystring = require('querystring');
+var winston = require('../../config/winston');
+
+// Headers that must not be forwarded between the two connections
+const HOP_BY_HOP_HEADERS = [
+  'connection',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'te',
+  'trailer',
+  'transfer-encoding',
+  'upgrade'
+];
+
+function forwardableRequestHeaders(req) {
+  let headers = Object.assign({}, req.headers);
+  HOP_BY_HOP_HEADERS.forEach((header) => {
+    delete headers[header];
+  });
+  delete headers['host'];
+  delete headers['content-length'];
+  return headers;
+}
+
+function requestBody(req, headers) {
+  // A request without content-length nor transfer-encoding has no body at all
+  if (req.headers['content-length'] === undefined && req.headers['transfer-encoding'] === undefined) {
+    return undefined;
+  }
+
+  // body-parser sets _body when it already consumed the stream (json / urlencoded),
+  // in that case the parsed body must be serialized again
+  if (req._body) {
+    let contentType = req.headers['content-type'] || '';
+    if (contentType.indexOf('application/x-www-form-urlencoded') > -1) {
+      return querystring.stringify(req.body);
+    }
+    return req.body;
+  }
+
+  // Everything else (multipart, binary, ...) is piped as it is
+  return req;
+}
+
+router.use(function (req, res) {
+
+  let baseUrl = process.env.DOCSPARSE_BASEURL;
+
+  if (!baseUrl) {
+    winston.error("DOCSPARSE_BASEURL is not set. Unable to forward the request to docsparse");
+    return res.status(503).send({ success: false, msg: "Docsparse module is not configured. Set the DOCSPARSE_BASEURL environment variable" });
+  }
+
+  // req.url is the part of the url following the module route, query string included
+  let url = baseUrl.replace(/\/+$/, '') + req.url;
+  let headers = forwardableRequestHeaders(req);
+
+  winston.debug("[DOCSPARSE] " + req.method + " " + url);
+
+  axios({
+    url: url,
+    method: req.method,
+    headers: headers,
+    data: requestBody(req, headers),
+    responseType: 'stream',
+    decompress: false,
+    maxContentLength: Infinity,
+    maxBodyLength: Infinity,
+    validateStatus: () => true
+  }).then((response) => {
+
+    winston.debug("[DOCSPARSE] response status " + response.status + " for " + url);
+
+    Object.keys(response.headers).forEach((header) => {
+      if (HOP_BY_HOP_HEADERS.indexOf(header.toLowerCase()) === -1) {
+        res.setHeader(header, response.headers[header]);
+      }
+    });
+
+    res.status(response.status);
+    response.data.pipe(res);
+
+  }).catch((err) => {
+    winston.error("[DOCSPARSE] Error forwarding the request to " + url + ": " + err.message);
+    res.status(502).send({ success: false, msg: "Error forwarding the request to docsparse", error: err.message });
+  })
+
+});
+
+module.exports = router;

package/pubmodules/docsparse/index.js

@@ -0,0 +1,3 @@
+const docsparseRoute = require("./docsparseRoute");
+
+module.exports = { docsparseRoute: docsparseRoute }

package/pubmodules/pubModulesManager.js

@@ -76,6 +76,9 @@ class PubModulesManager {
         this.dialogFlow = undefined;
 
         this.analyticsPublisher = undefined;
+
+        this.docsparse = undefined;
+        this.docsparseRoute = undefined;
     }
 
   
@@ -170,6 +173,12 @@ class PubModulesManager {
             winston.info("ModulesManager trigger controller loaded");       
         }
 
+        if (this.docsparseRoute) {
+            // docsparse apis have no authentication on their own: the server authenticates the caller before forwarding
+            app.use('/:projectid/docsparse', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], this.docsparseRoute);
+            winston.info("ModulesManager docsparse controller loaded");
+        }
+
     }
 
    
@@ -609,6 +618,21 @@ class PubModulesManager {
                 winston.info("PubModulesManager error initializing analyticsPublisher module", err);
             }
         }
+
+        try {
+            this.docsparse = require('./docsparse');
+            winston.debug("this.docsparse: " + this.docsparse);
+
+            this.docsparseRoute = this.docsparse.docsparseRoute;
+
+            winston.info("PubModulesManager initialized docsparse.");
+        } catch(err) {
+            if (err.code == 'MODULE_NOT_FOUND') {
+                winston.info("PubModulesManager init docsparse module not found");
+            } else {
+                winston.info("PubModulesManager error initializing docsparse module", err);
+            }
+        }
     }
 
     start() {