@muze-nl/simplystore 0.8.1 → 0.9.0

package/README.md

@@ -2,9 +2,9 @@
 
 SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. All data is read into memory and is available to these javascript queries without needing (or allowing) disk access or indexes.
 
-[JSONTag](https://github.com/poef/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags.
+[JSONTag](https://github.com/muze-nl/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags.
 Javascript queries are run in a [VM2](https://www.npmjs.com/package/vm2) sandbox. 
-You can query data using the [array-where-select](https://www.npmjs.com/package/array-where-select) extension.
+You can query data using the [jaqt](https://github.com/muze-nl/jaqt/)  library.
 
 Note: _There are known security issues in VM2, so the project will switch to V8-isolate. For now make sure SimplyStore is not publically accessible, by adding an api gateway in front of it for example_
 
@@ -168,4 +168,4 @@ In addition, SimplyStore is meant to be a real-world testcase for JSONTag.
 [MIT](LICENSE) © Muze.nl
 
 ## Contributions
-Contributions are welcome, but make sure that all code is MIT licensed. If you want to send a merge request, please make sure that there is a ticket that shows the bug/feature and reference it. If you find any problem, please do file a ticket, but you should not expect a timely resolution. This project is still very experimental, don't use it in production unless you are ready to fix problems yourself.
+Contributions are welcome, but make sure that all code is MIT licensed. If you want to send a merge request, please make sure that there is a ticket that shows the bug/feature and reference it. If you find any problem, please do file a ticket, but you should not expect a timely resolution. This project is still very experimental, don't use it in production unless you are ready to fix problems yourself.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.8.1",
+	"version": "0.9.0",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -16,7 +16,7 @@
 	"bugs": "https://github.com/simplyedit/simplystore/issues",
 	"homepage": "https://github.com/simplyedit/simplystore#readme",
 	"dependencies": {
-		"@muze-nl/jaqt": "^0.10.2",
+		"@muze-nl/jaqt": "^0.10.3",
 		"@muze-nl/jsontag": "^0.10.2",
 		"@muze-nl/od-jsontag": "^0.4.1",
 		"codemirror": "^6.0.1",

package/src/server.mjs

@@ -40,7 +40,6 @@ async function main(options) {
     }))
 
     let status = loadCommandStatus(commandStatus)
-    let commandQueue = loadCommandLog(status, commandLog)
 
     try {
         let data = await loadData(Array.from(status.keys())) // command id's (keys) are used to generate filenames of changes
@@ -51,6 +50,8 @@ async function main(options) {
         process.exit(1)
     }
 
+    let commandQueue = loadCommandLog(status, commandLog)
+
     const queryWorkerInitTask = () => { 
         return {
             name: 'init',
@@ -77,6 +78,14 @@ async function main(options) {
         console.error(`Port ${port} is already occupied, aborting.`)
         process.exit()
     } catch(err) {
+        let result
+        do {
+            try {
+                result = await runNextCommand()
+            } catch(err) {
+                // console.log(err) // ignore errors here, already logged to console
+            }
+        } while(result)
         server.listen(port, () => {
             console.log('SimplyStore listening on port '+port)
             let used = Math.round(process.memoryUsage().rss / 1024 / 1024);
@@ -115,10 +124,18 @@ async function main(options) {
             let lines = log.split("\n").filter(Boolean)
             for(let line of lines) {
                 let command = JSONTag.parse(line)
-                let state = status.get(command.id)
+                let state = status.get(command.id)?.status
                 switch(state) {
                     case 'accepted': // enqueue
-                        commands.push(command)
+                        commands.push({
+                            id: command.id,
+                            command: line,
+                            request: null,
+                            meta,
+                            data:jsontagBuffers,
+                            commandsFile,
+                            datafile
+                        })
                         break;
                     case 'done': // do nothing
                         break;
@@ -238,8 +255,10 @@ async function main(options) {
                 commandsFile,
                 datafile                
             })
-
-            runNextCommand()
+            let result
+            do {
+                result = await runNextCommand()
+            } while(result)
         } catch(err) {
             let s = {code:err.code||500, status:'failed', message:err.message, details:err.details}
             status.set(commandId, s)
@@ -265,67 +284,80 @@ async function main(options) {
     }
 
     async function runNextCommand() {
-        let command = commandQueue.shift()
-        if (command) {
-            let start = (resolve, reject) => {
-                if (!commandWorkerInstance) {
+        return new Promise(async (mainResolve, mainReject) => {
+            if (commandWorkerInstance) {
+                mainReject('commandWorker already running')
+                return
+            }
+            let command = commandQueue.shift()
+            if (command) {
+                console.log('starting command',command.id)
+                let start = (resolve, reject) => {
                     commandWorkerInstance = new Worker(commandWorker)
+                    commandWorkerInstance.on('message', async result => {
+                        await commandWorkerInstance.terminate()
+                        commandWorkerInstance = null
+                        resolve(result)
+                    })
+                    commandWorkerInstance.on('error', async error => {
+                        await commandWorkerInstance.terminate()
+                        commandWorkerInstance = null
+                        reject(error)
+                    })
+                    commandWorkerInstance.postMessage(command)
                 }
-                commandWorkerInstance.on('message', result => {
-                    resolve(result)
-                    runNextCommand()
-                })
-                commandWorkerInstance.on('error', error => {
-                    reject(error)
-                    runNextCommand()
-                })
-                commandWorkerInstance.postMessage(command)
-            }
-            start(
-                // resolve()
-                (data) => {
-                    let s
-                    if (!data || (data.code>=300 && data.code<=499)) {
-                        console.error('ERROR: SimplyStore cannot run command ', command.id, data)
-                        if (!data?.code) {
-                            s = {code: 500, status: "failed"}
+                start(
+                    // resolve()
+                    (data) => {
+                        let s
+                        if (!data || (data.code>=300 && data.code<=499)) {
+                            console.error('ERROR: SimplyStore cannot run command ', command.id, data)
+                            if (!data?.code) {
+                                s = {code: 500, status: "failed"}
+                            } else {
+                                s = {code: data.code, status: "failed", message: data.message, details: data.details}
+                            }
+                            status.set(command.id, s)
+                            appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                            mainReject(s)
                         } else {
-                            s = {code: data.code, status: "failed", message: data.message, details: data.details}
+                            s = {code: 200, status: "done"}
+                            status.set(command.id, s)
+                            if (data.data) { // data has changed, commands may do other things instead of changing data
+                                jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
+                                Object.assign(meta, data.meta)
+                                queryWorkerPool.update({
+                                    name: 'update',
+                                    req: {
+                                        body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
+                                        meta
+                                    }
+                                })
+                            }
+                            appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                            mainResolve(s)
                         }
+                    }, 
+                    //reject()
+                    (error) => {
+                        let s = {status: "failed", code: error.code, message: error.message, details: error.details}
                         status.set(command.id, s)
-                    } else {
-                        s = {code: 200, status: "done"}
-                        status.set(command.id, s)
-                        if (data.data) { // data has changed, commands may do other things instead of changing data
-                            jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
-                            Object.assign(meta, data.meta)
-                            queryWorkerPool.update({
-                                name: 'update',
-                                req: {
-                                    body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
-                                    meta
-                                }
-                            })
-                        }
+                        appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                        console.log('command error', command.id, error)
+                        mainReject(s)
                     }
-                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
-                }, 
-                //reject()
-                (error) => {
-                    console.error(error)
-                    let s = {status: "failed", code: error.code, message: error.message, details: error.details}
-                    status.set(command.id, s)
-                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                )
+            } else {
+                console.log('no pending commands')
+                // this code can never be triggered from the post(/command/) route, since it always adds a command to the queue
+                // so you can only get here from commandWorkerInstance.on() route
+                // which means that the commandWorkerInstance has finished running the previous command
+                if (commandWorkerInstance) {
+                    await commandWorkerInstance.terminate()
                 }
-            )
-        } else {
-            // this code can never be triggered from the post(/command/) route, since it always adds a command to the queue
-            // so you can only get here from commandWorkerInstance.on() route
-            // which means that the commandWorkerInstance has finished running the previous command
-            await commandWorkerInstance.terminate()
-            commandWorkerInstance.unref() // @FIXME is this needed?
-            commandWorkerInstance = null  // @FIXME or this?
-        }
+                mainResolve(false)
+            }
+        })
     }
 
     function checkCommand(req, res) {
@@ -368,13 +400,12 @@ async function main(options) {
             sendResponse({code:422, body: JSON.stringify(error)}, res)
             return false      
         }
-        appendFile(commandLog, JSONTag.stringify(command))
+        appendFile(commandLog, JSONTag.stringify(command)) //FIXME: this loses request data
         appendFile(commandStatus, JSONTag.stringify(commandOK))
         status.set(command.id, commandOK) 
         sendResponse({code: 202, body: JSON.stringify(commandOK)}, res)
         return command.id
     }
-
 }
 
 function sendResponse(response, res) {
@@ -435,4 +466,4 @@ function handleWebRequest(req,res,options)
     } else {
         res.sendFile('/index.html', fileOptions)
     }
-}
+}