@tiledesk/tiledesk-server 2.5.3 → 2.7.0

package/test/kbRoute.js

@@ -1,10 +1,11 @@
 //During the test the env variable is set to test
 process.env.NODE_ENV = 'test';
+process.env.GPTKEY = "fakegptkey";
 
 var userService = require('../services/userService');
 var projectService = require('../services/projectService');
 
-let log = true;
+let log = false;
 
 //Require the dev-dependencies
 let chai = require('chai');
@@ -37,7 +38,7 @@ describe('KbRoute', () => {
                         name: "example_name5",
                         type: "url",
                         source: "https://www.exampleurl5.com",
-                        content: ""
+                        content: "",
                     }
 
                     chai.request(server)
@@ -45,36 +46,106 @@ describe('KbRoute', () => {
                         .auth(email, pwd)
                         .send(kb) // can be empty
                         .end((err, res) => {
+                            
+                            if (err) { console.error("err: ", err); }
                             if (log) { console.log("create kb res.body: ", res.body); }
+                            
                             res.should.have.status(200);
-                            // res.body.should.be.a('object');
-                            // expect(res.body.id_project).to.equal(savedProject._id.toString());
+                            res.body.should.be.a('object');
 
                             done();
-                            // chai.request(server)
-                            //     .get('/' + savedProject._id + "/kbsettings")
-                            //     .auth(email, pwd)
-                            //     .end((err, res) => {
-                            //         if (log) { console.log("get kbsettings res.body: ", res.body); }
-                            //         res.should.have.status(200);
-                            //         res.body.should.be.a('object');
-                            //         expect(res.body.id_project).to.equal(savedProject._id.toString())
-                            //         expect(res.body.maxKbsNumber).to.equal(3);
-                            //         expect(res.body.maxPagesNumber).to.equal(1000);
-                            //         expect(res.body.kbs).is.an('array').that.is.empty;
+                   
+
+                        })
+
+                });
+            });
+
+        }).timeout(10000);
+
+        // logic in standby
+        // it('createNewKb-namespaceNotBelongsProject', (done) => {
+
+        //     var email = "test-signup-" + Date.now() + "@email.com";
+        //     var pwd = "pwd";
+
+        //     userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+        //         projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+        //             let kb = {
+        //                 name: "example_name5",
+        //                 type: "url",
+        //                 source: "https://www.exampleurl5.com",
+        //                 content: "",
+        //                 namespace: "fakenamespace"
+        //             }
+
+        //             chai.request(server)
+        //                 .post('/' + savedProject._id + '/kb')
+        //                 .auth(email, pwd)
+        //                 .send(kb) // can be empty
+        //                 .end((err, res) => {
+                            
+        //                     if (err) { console.error("err: ", err); }
+        //                     if (log) { console.log("create kb res.body: ", res.body); }
+                            
+        //                     res.should.have.status(403);
+        //                     res.body.should.be.a('object');
+        //                     expect(res.body.success).to.equal(false);
+        //                     expect(res.body.error).to.equal("Not allowed. The namespace does not belong to the current project.");
+
+        //                     done();
+                   
+
+        //                 })
+
+        //         });
+        //     });
+
+        //}).timeout(10000);
+
+        it('createNewKb-replaceNamespace', (done) => {
 
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    let kb = {
+                        name: "example_name5",
+                        type: "url",
+                        source: "https://www.exampleurl5.com",
+                        content: "",
+                        namespace: "fakenamespace"
+                    }
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb')
+                        .auth(email, pwd)
+                        .send(kb) // can be empty
+                        .end((err, res) => {
+                            
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("create kb res.body: ", res.body); }
+                            
+                            res.should.have.status(200);
+                            res.body.should.be.a('object');
+                            expect(res.body.value.namespace).not.equal("fakenamespace");
+                            expect(res.body.value.namespace).to.equal(savedProject._id.toString());
 
-                            //     })
+                            done();
+                   
 
                         })
 
                 });
             });
 
-        });
+        }).timeout(10000);
 
         it('getWithQueries', (done) => {
-            
+
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
 
@@ -136,7 +207,7 @@ describe('KbRoute', () => {
                                                         .get('/' + savedProject._id + "/kb" + query)
                                                         .auth(email, pwd)
                                                         .end((err, res) => {
-                                                            console.log("getall res.body: ", res.body);
+                                                            if (log) { console.log("getall res.body: ", res.body); }
                                                             res.should.have.status(200);
                                                             res.body.should.be.a('object');
                                                             res.body.kbs.should.be.a('array');
@@ -153,7 +224,7 @@ describe('KbRoute', () => {
                                                             done();
 
                                                         })
-            
+
                                                 })
                                         }, 1000)
                                     })
@@ -220,6 +291,246 @@ describe('KbRoute', () => {
 
         });
 
+        // it('scrapeSingle-namespaceNotBelongsProject', (done) => {
+
+        //     var email = "test-signup-" + Date.now() + "@email.com";
+        //     var pwd = "pwd";
+
+        //     userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+        //         projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+        //             let kb = {
+        //                 name: "example_name6",
+        //                 type: "url",
+        //                 source: "https://www.exampleurl6.com",
+        //                 content: ""
+        //             }
+
+        //             chai.request(server)
+        //                 .post('/' + savedProject._id + '/kb')
+        //                 .auth(email, pwd)
+        //                 .send(kb) // can be empty
+        //                 .end((err, res) => {
+        //                     if (log) { console.log("create kb res.body: ", res.body); }
+        //                     res.should.have.status(200);
+
+        //                     let kbid = res.body.value._id;
+        //                     console.log("kbid: ", kbid)
+        //                     chai.request(server)
+        //                         .post('/' + savedProject._id + "/kb/scrape/single")
+        //                         .auth(email, pwd)
+        //                         .send({ id: kbid })
+        //                         .end((err, res) => {
+        //                             if (log) { console.log("single scrape res.body: ", res.body); }
+        //                             //res.should.have.status(200);
+        //                             // res.body.should.be.a('object');
+        //                             // expect(res.body.id_project).to.equal(savedProject._id.toString())
+        //                             // expect(res.body.maxKbsNumber).to.equal(3);
+        //                             // expect(res.body.maxPagesNumber).to.equal(1000);
+        //                             // expect(res.body.kbs).is.an('array').that.is.empty;
+        //                             done();
+
+        //                         })
+
+
+        //                     // res.body.should.be.a('object');
+        //                     // expect(res.body.id_project).to.equal(savedProject._id.toString());
+
+
+
+
+        //                 })
+        //         });
+        //     });
+        // });
+
+        it('multiadd', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb/multi')
+                        .auth(email, pwd)
+                        .set('Content-Type', 'text/plain')
+                        .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './kbUrlsList.txt')), 'kbUrlsList.txt')
+                        .end((err, res) => {
+
+                            // console.log("res.body: ", res.body)
+                            res.should.have.status(200);
+                            expect(res.body.length).to.equal(4)
+                            
+                            done();
+
+                            // setTimeout(() => {
+
+                            //     chai.request(server)
+                            //         .post('/' + savedProject._id + '/kb/multi')
+                            //         .auth(email, pwd)
+                            //         .set('Content-Type', 'text/plain')
+                            //         .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './kbUrlsList.txt')), 'kbUrlsList.txt')
+                            //         .end((err, res) => {
+
+                            //             // console.log("res.body: ", res.body);
+                            //             res.should.have.status(200);
+                            //             expect(res.body.length).to.equal(4)
+
+                            //             done()
+                            //         })
+                            // }, 2000)
+
+                        })
+
+                });
+            });
+
+        }).timeout(10000)
+
+        it('multiadd-fail', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb/multi')
+                        .auth(email, pwd)
+                        .set('Content-Type', 'text/plain')
+                        .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './kbUrlsList.txt')), 'kbUrlsList.txt')
+                        .end((err, res) => {
+
+                            // console.log("res.body: ", res.body)
+                            res.should.have.status(200);
+                            expect(res.body.length).to.equal(4)
+
+                            setTimeout(() => {
+
+                                chai.request(server)
+                                    .post('/' + savedProject._id + '/kb/multi')
+                                    .auth(email, pwd)
+                                    .set('Content-Type', 'text/plain')
+                                    .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './kbUrlsList.txt')), 'kbUrlsList.txt')
+                                    .end((err, res) => {
+
+                                        // console.log("res.body: ", res.body);
+                                        res.should.have.status(200);
+                                        expect(res.body.length).to.equal(4)
+
+                                        done()
+                                    })
+                            }, 2000)
+
+                        })
+
+                });
+            });
+
+        }).timeout(10000)
+
+        // it('tooManyUrls', (done) => {
+
+        //     var email = "test-signup-" + Date.now() + "@email.com";
+        //     var pwd = "pwd";
+
+        //     userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+        //         projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+        //             chai.request(server)
+        //                 .post('/' + savedProject._id + '/kb/multi')
+        //                 .auth(email, pwd)
+        //                 .set('Content-Type', 'text/plain')
+        //                 .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './TooManykbUrlsList.txt')), 'TooManykbUrlsList.txt')
+        //                 .end((err, res) => {
+
+        //                     // console.log("res.body: ", res.body)
+        //                     res.should.have.status(403);
+        //                     expect(res.body.success).to.equal(false);
+        //                     expect(res.body.error).to.equal("Too many urls. Can't index more than 300 urls at a time.");
+
+        //                     done()
+
+        //                 })
+
+        //         });
+        //     });
+
+        // })
+
+        // logic in standby
+        // it('askkb-namespaceNotBelongsProject', (done) => {
+
+        //     var email = "test-signup-" + Date.now() + "@email.com";
+        //     var pwd = "pwd";
+
+        //     userService.signup(email, pwd, "Test Firstname", "Test Lastname").then((savedUser) => {
+        //         projectService.create("test-kb-qa", savedUser._id).then((savedProject) => {
+
+        //             /**
+        //              * README
+        //              * Namespace should be equal to savedProject._id;
+        //              * A generic mongodb ID (like user id) is used instead for test porpouse
+        //              */
+        //             chai.request(server)
+        //                 .post('/' + savedProject._id + "/kb/qa")
+        //                 .auth(email, pwd)
+        //                 .send({ model: "gpt-4", namespace: savedUser._id, question: "sample question"})
+        //                 .end((err, res) => {
+
+        //                     if (err) { console.log("error: ", err) };
+        //                     if (log) { console.log("res.body: ", res.body) };
+
+        //                     res.should.have.status(403); 
+        //                     res.body.should.be.a('object');
+        //                     expect(res.body.success).to.equal(false);
+        //                     expect(res.body.error).to.equal("Not allowed. The namespace does not belong to the current project.");
+                            
+        //                     done();                            
+        //                 })
+        //         })
+        //     })
+        // }).timeout(10000)
+
+
+        it('sitemap', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test lastname").then(function (savedUser) {
+                projectService.create("test-faqkb-create", savedUser._id).then(function (savedProject) {
+
+                    chai.request(server)
+                        .post('/' + savedProject._id + '/kb/sitemap')
+                        .auth(email, pwd)
+                        // .send({ sitemap: "https://www.wired.it/sitemap.xml" })
+                        .send({ sitemap: "https://gethelp.tiledesk.com/sitemap.xml" })
+                        .end((err, res) => {
+
+                            if (err) { console.log("error: ", err) };
+                            if (log) { console.log("res.body: ", res.body) }
+
+                            console.log("error: ", err)
+                            console.log("res.body: ", res.body)
+
+                            console.log("sites length: ", res.body.sites.length)
+                            res.should.have.status(200);
+                            res.body.should.be.a('object');
+                            res.body.sites.should.be.a('array');
+
+                            done();
+
+                        })
+
+                });
+            });
+
+        }).timeout(10000)
+
     })
 });
 

package/test/kbUrlsList.txt

@@ -0,0 +1,4 @@
+https://gethelp.tiledesk.com/articles/article1
+https://gethelp.tiledesk.com/articles/article2
+https://gethelp.tiledesk.com/articles/article3
+https://gethelp.tiledesk.com/articles/article4

package/utils/jobs-worker-queue-manager/JobManagerV2.js

@@ -0,0 +1,146 @@
+var QueueManager = require("./queueManagerClassV2");
+
+class JobManager {
+    constructor(queueUrl, options) {
+        this.queueUrl = queueUrl;
+        // this.queueManager = null;
+        this.debug = false;    
+        if (options && options.debug!=undefined) {
+            this.debug = options.debug;
+        }
+
+        this.info = true;
+        this.queueManager = new QueueManager(this.queueUrl, options);
+
+        this.sendingJobs = [];
+        this.queuePublisherConnected = false;
+        this.queueConsumerConnected = false;
+
+    }
+    connectAndStartPublisher(callback) {
+        var that = this;
+        if (this.info) {console.log("[JobWorker] JobManager publisher started");}
+        
+        this.queueManager.connect(function() {
+            if (that.debug) {console.log("[JobWorker] Queue started");}
+            that.queuePublisherConnected = true;
+
+            that.queueManager.startPublisher(function() {
+                if (that.debug) {console.log("[JobWorker] Queue that.sendingJobs.length",that.sendingJobs.length);}
+                if (that.sendingJobs.length > 0) {
+                    for (var i = 0; i<that.sendingJobs.length; i++) {
+                        if (that.debug) {console.log("[JobWorker] Queue that.sendingJobs[i]",that.sendingJobs[i]);}
+                        // that.queueManager.send(that.sendingJobs[i].toString(), "functions");
+                        that.queueManager.sendJson(that.sendingJobs[i], "functions");
+                        // that.sendingJobs[i]();
+                    }
+                    //that.sendingJob = [];
+                    that.sendingJobs.length = 0; //empty array that.sendingJobs
+                    if (that.debug) {console.log("[JobWorker] that.sendingJobs[i] cleared",that.sendingJobs);}
+
+                } 
+
+                if (callback) {
+                    callback();
+                }
+            });
+        });
+    }
+
+    diconnectPublisher(callback) {
+
+        if (this.info) {console.log("[JobWorker] JobManager publisher disconnected");}
+
+        this.queueManager.close((err) => {
+            if (err) {
+                console.error("JobManager: error closing connection: ", err);
+            }
+            callback(err);
+        })
+        
+
+    }
+
+    connectAndStartWorker() {
+        var that = this;
+        if (this.info) {console.log("[JobWorker] JobManager worker started");}
+        
+        this.queueManager.connect(function() {
+            if (that.debug) {console.log("[JobWorker] Queue started");}
+            that.queueConsumerConnected = true;
+
+            that.queueManager.startWorker(function() {
+                if (that.debug) {console.log("[JobWorker] Queue that.queueManager.startWorker");}               
+            });
+        });
+    }
+
+    publish(payload, callback) {
+        
+        var packet = {payload: payload}
+        // if (this.debug) {console.log("JobManager this.queueConnected",this.queueStarted);
+        if (this.queuePublisherConnected == true) {
+
+            if (this.debug) {console.log("[JobWorker] JobManager  this.queuePublisherConnected == true");}
+            this.queueManager.sendJson(packet, "functions", (err, ok) => {
+                if (err) {
+                    console.error("sendJson error: ", err);
+                } else {
+                    if (this.debug) { console.log("sendJson ok"); };
+                }
+                callback(err, ok);
+            });
+
+            // this.queueManager.on(fn);
+        } 
+        // else {
+        //     if (this.debug) {console.log("[JobWorker] JobManager  this.queuePublisherConnected == false");}
+            
+        //     this.connectAndStartPublisher();
+        //     this.sendingJobs.push(packet);
+
+        // }
+       
+    }
+
+
+    //Deprecated
+    schedule(fn, payload) {
+        
+        var func = {function: fn.toString(), payload: payload}
+        // if (this.debug) {console.log("JobManager this.queueConnected",this.queueStarted);
+        if (this.queuePublisherConnected == true) {
+            if (this.debug) {console.log("[JobWorker] JobManager  this.queuePublisherConnected == true");}
+            this.queueManager.send(func, "functions");
+
+            // this.queueManager.on(fn);
+        } else {
+            if (this.debug) {console.log("[JobWorker] JobManager  this.queuePublisherConnected == false");}
+            
+            this.connectAndStartPublisher();
+            this.sendingJobs.push(func);
+
+        }
+       
+    }
+
+    run(callback) {
+        // if (this.queueConsumerConnected == true) {
+        //     if (this.debug) {console.log("JobManager  this.queueConsumerConnected == true");
+        //     this.queueManager.send(fn.toString(), "functions");
+
+        // } else {
+            if (this.debug) {console.log("[JobWorker] JobManager connectAndStartWorker");}
+            this.connectAndStartWorker();
+
+            if (callback) {
+                if (this.debug) {console.log("[JobWorker] JobManager  callback", callback);}
+                this.queueManager.on(callback);
+            }
+        // }
+    }
+}
+
+
+
+module.exports = JobManager;