@muze-nl/simplystore 0.7.4 → 0.9.0

package/README.md

@@ -2,9 +2,9 @@
 
 SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. All data is read into memory and is available to these javascript queries without needing (or allowing) disk access or indexes.
 
-[JSONTag](https://github.com/poef/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags.
+[JSONTag](https://github.com/muze-nl/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags.
 Javascript queries are run in a [VM2](https://www.npmjs.com/package/vm2) sandbox. 
-You can query data using the [array-where-select](https://www.npmjs.com/package/array-where-select) extension.
+You can query data using the [jaqt](https://github.com/muze-nl/jaqt/)  library.
 
 Note: _There are known security issues in VM2, so the project will switch to V8-isolate. For now make sure SimplyStore is not publically accessible, by adding an api gateway in front of it for example_
 
@@ -168,4 +168,4 @@ In addition, SimplyStore is meant to be a real-world testcase for JSONTag.
 [MIT](LICENSE) © Muze.nl
 
 ## Contributions
-Contributions are welcome, but make sure that all code is MIT licensed. If you want to send a merge request, please make sure that there is a ticket that shows the bug/feature and reference it. If you find any problem, please do file a ticket, but you should not expect a timely resolution. This project is still very experimental, don't use it in production unless you are ready to fix problems yourself.
+Contributions are welcome, but make sure that all code is MIT licensed. If you want to send a merge request, please make sure that there is a ticket that shows the bug/feature and reference it. If you find any problem, please do file a ticket, but you should not expect a timely resolution. This project is still very experimental, don't use it in production unless you are ready to fix problems yourself.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.7.4",
+	"version": "0.9.0",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -16,9 +16,9 @@
 	"bugs": "https://github.com/simplyedit/simplystore/issues",
 	"homepage": "https://github.com/simplyedit/simplystore#readme",
 	"dependencies": {
-		"@muze-nl/jaqt": "^0.10.2",
-		"@muze-nl/jsontag": "^0.9.11",
-		"@muze-nl/od-jsontag": "^0.3.4",
+		"@muze-nl/jaqt": "^0.10.3",
+		"@muze-nl/jsontag": "^0.10.2",
+		"@muze-nl/od-jsontag": "^0.4.1",
 		"codemirror": "^6.0.1",
 		"express": "^4.18.1",
 		"json-pointer": "^0.6.2",

package/src/command-worker-module.mjs

@@ -1,6 +1,6 @@
 import JSONTag from '@muze-nl/jsontag'
 import {getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
-import parse from '@muze-nl/od-jsontag/src/parse.mjs'
+import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 import writeFileAtomic from 'write-file-atomic'
 
@@ -13,10 +13,12 @@ let metaProxy = {
     index: {
     }
 }
+const parser = new Parser()
+parser.immutable = false
 
 export const metaIdProxy = {
     forEach: (callback) => {
-        meta.index.id.forEach((ref,id) => {
+        parser.meta.index.id.forEach((ref,id) => {
             callback({
                 deref: () => {
                     return resultArr[ref]
@@ -25,19 +27,19 @@ export const metaIdProxy = {
         })
     },
     set: (id,ref) => {
-        if (!meta.index.id.has(id)) {
+        if (!parser.meta.index.id.has(id)) {
             if (ref[getIndex]) {
-                meta.index.id.set(id, ref[getIndex])
+                parser.meta.index.id.set(id, ref[getIndex])
             } else {
                 throw new Error('cannot set index.id for non-proxy')
             }
         } else {
-            let line = meta.index.id.get(id)
+            let line = parser.meta.index.id.get(id)
             resultArr[line] = ref
         }
     },
     get: (id) => {
-        let index = meta.index.id.get(id)
+        let index = parser.meta.index.id.get(id)
         if (index || index===0) {
             return {
                 deref: () => {
@@ -47,7 +49,7 @@ export const metaIdProxy = {
         }
     },
     has: (id) => {
-        return meta.index.id.has(id)
+        return parser.meta.index.id.has(id)
     }
 }
 
@@ -59,8 +61,11 @@ const metaReadProxy = {
 }
 
 export async function initialize(task) {
+    if (task.meta) {
+        parser.meta = task.meta
+    }
     for(let jsontag of task.data) {
-        dataspace = parse(jsontag, task.meta, false) // false means mutable
+        dataspace = parser.parse(jsontag)
     }
     resultArr = dataspace[resultSet]
     meta = task.meta
@@ -88,7 +93,6 @@ export default async function runCommand(commandStr, request) {
             let time = Date.now()
             commands[task.name](dataspace, task, request, metaProxy)
             //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
-            JSONTag.setAttribute(dataspace, 'command', task.id)
         
             const uint8sab = serialize(dataspace, {meta, changes: true}) // serialize only changes
             response.data = uint8sab

package/src/load-worker.mjs

@@ -1,9 +1,10 @@
 import { parentPort } from 'node:worker_threads'
 import JSONTag from '@muze-nl/jsontag'
-import parse from '@muze-nl/od-jsontag/src/parse.mjs'
+import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
 import fs from 'fs'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 
+const parser = new Parser()
 parentPort.on('message', (files) => {
 	let meta = {
 		index: {
@@ -17,22 +18,20 @@ parentPort.on('message', (files) => {
 	let count = 0
 	let data
 	let jsontag
-	let tempMeta = {}
 	let datafile = files.dataFile
 	let commands = files.commands
 	commands.push('done')
 	do {
 		if (fs.existsSync(datafile)) {
 			jsontag = fs.readFileSync(datafile)
-			data = parse(jsontag, tempMeta) // tempMeta is needed to combine the resultArray, using meta conflicts with meta.index.id
+			data = parser.parse(jsontag)
 			count++
 		}
 		datafile = basefile + '.' + commands.shift() + '.' + extension
 	} while(commands.length)
-	meta.parts = count
 	if (files.schemaFile) {
 		jsontag = fs.readFileSync(files.schemaFile, 'utf-8')
-		meta.schema = JSONTag.parse(jsontag, null, tempMeta)
+		meta.schema = JSONTag.parse(jsontag)
 	}
 
 	const sab = serialize(data, {meta})

package/src/query-worker-module.mjs

@@ -2,66 +2,28 @@ import pointer from 'json-pointer'
 import {VM} from 'vm2'
 import { memoryUsage } from 'node:process'
 import JSONTag from '@muze-nl/jsontag'
-import * as odJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
 import {source} from '@muze-nl/od-jsontag/src/symbols.mjs'
-import parse from '@muze-nl/od-jsontag/src/parse.mjs'
+import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
 import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,distinct} from '@muze-nl/jaqt'
 import process from 'node:process'
 
 let dataspace
-let meta = {}
 let metaProxy = {
     index: {
     }
 }
 
-function protect(target) {
-    if (target[source]) {
-        throw new Error('Data is immutable')
-    }
-}
-
-const myJSONTag = {
-    getAttribute: odJSONTag.getAttribute,
-    getAttributes: odJSONTag.getAttributes,
-    getType: odJSONTag.getType,
-    getTypeString: odJSONTag.getTypeString,
-    setAttribute: (target, name, value) => {
-        protect(target)
-        return odJSONTag.setAttribute(target, name, value)
-    },
-    setType: (target, type) => {
-        protect(target)
-        return odJSONTag.setType(target, type)
-    },
-    setAttributes: (target, attributes) => {
-        protect(target)
-        return odJSONTag.setAttributes(target, attributes)
-    },
-    addAttribute: (target, name, value) => {
-        protect(target)
-        return odJSONTag.addAttribute(target, name, value)
-    },
-    removeAttribute: (target, name) => {
-        protect(target)
-        return odJSONTag.removeAttribute(target, name)
-    },
-    getAttributesString: odJSONTag.getAttributesString,
-    isNull: odJSONTag.isNull,
-    clone: JSONTag.clone,
-    Link: JSONTag.Link,
-    Null: JSONTag.Null
-}
+const parser = new Parser()
 
 const metaIdProxy = {
     get: (id) => {
-        let index = meta.index.id.get(id)
+        let index = parser.meta.index.id.get(id)
         if (index || index===0) {
-            return meta.resultArray[index]
+            return parser.meta.resultArray[index]
         }
     },
     has: (id) => {
-        return meta.index.id.has(id)
+        return parser.meta.index.id.has(id)
     }
 }
 
@@ -70,27 +32,27 @@ const tasks = {
         if (task.req.access) {
             task.req.access = await import(task.req.access)
             task.req.access = task.req.access.default
-            meta.access = task.req.access
+            parser.meta.access = task.req.access
         }
         if (task.req.meta.index) {
-            meta.index = task.req.meta.index
+            parser.meta.index = task.req.meta.index
         }
         if (task.req.meta.schema) {
-            meta.schema = task.req.meta.schema
+            parser.meta.schema = task.req.meta.schema
         }
         for (let sab of task.req.body) { //body contains an array of sharedArrayBuffers with initial data and changes
-            dataspace = parse(sab, meta)
+            dataspace = parser.parse(sab)
         }
         metaProxy.index.id = metaIdProxy
-        metaProxy.schema = meta.schema
+        metaProxy.schema = parser.meta.schema
         //@TODO: add meta.index.references? and baseURL
         return true
     },
     update: async (task) => {
         if (task.req.meta.index) {
-            meta.index = task.req.meta.index
+            parser.meta.index = task.req.meta.index
         }
-        dataspace = parse(task.req.body, meta) //update only has a single changeset
+        dataspace = parser.parse(task.req.body) //update only has a single changeset
         return true
     },
     query: async (task) => {
@@ -139,7 +101,7 @@ export function runQuery(pointer, request, query) {
                 one,
                 distinct,
 //                    console: connectConsole(res),
-                JSONTag: myJSONTag,
+                JSONTag,
                 request
             },
             wasm: false

package/src/server.mjs

@@ -40,7 +40,6 @@ async function main(options) {
     }))
 
     let status = loadCommandStatus(commandStatus)
-    let commandQueue = loadCommandLog(status, commandLog)
 
     try {
         let data = await loadData(Array.from(status.keys())) // command id's (keys) are used to generate filenames of changes
@@ -51,6 +50,8 @@ async function main(options) {
         process.exit(1)
     }
 
+    let commandQueue = loadCommandLog(status, commandLog)
+
     const queryWorkerInitTask = () => { 
         return {
             name: 'init',
@@ -77,6 +78,14 @@ async function main(options) {
         console.error(`Port ${port} is already occupied, aborting.`)
         process.exit()
     } catch(err) {
+        let result
+        do {
+            try {
+                result = await runNextCommand()
+            } catch(err) {
+                // console.log(err) // ignore errors here, already logged to console
+            }
+        } while(result)
         server.listen(port, () => {
             console.log('SimplyStore listening on port '+port)
             let used = Math.round(process.memoryUsage().rss / 1024 / 1024);
@@ -115,10 +124,18 @@ async function main(options) {
             let lines = log.split("\n").filter(Boolean)
             for(let line of lines) {
                 let command = JSONTag.parse(line)
-                let state = status.get(command.id)
+                let state = status.get(command.id)?.status
                 switch(state) {
                     case 'accepted': // enqueue
-                        commands.push(command)
+                        commands.push({
+                            id: command.id,
+                            command: line,
+                            request: null,
+                            meta,
+                            data:jsontagBuffers,
+                            commandsFile,
+                            datafile
+                        })
                         break;
                     case 'done': // do nothing
                         break;
@@ -238,8 +255,10 @@ async function main(options) {
                 commandsFile,
                 datafile                
             })
-
-            runNextCommand()
+            let result
+            do {
+                result = await runNextCommand()
+            } while(result)
         } catch(err) {
             let s = {code:err.code||500, status:'failed', message:err.message, details:err.details}
             status.set(commandId, s)
@@ -265,67 +284,80 @@ async function main(options) {
     }
 
     async function runNextCommand() {
-        let command = commandQueue.shift()
-        if (command) {
-            let start = (resolve, reject) => {
-                if (!commandWorkerInstance) {
+        return new Promise(async (mainResolve, mainReject) => {
+            if (commandWorkerInstance) {
+                mainReject('commandWorker already running')
+                return
+            }
+            let command = commandQueue.shift()
+            if (command) {
+                console.log('starting command',command.id)
+                let start = (resolve, reject) => {
                     commandWorkerInstance = new Worker(commandWorker)
+                    commandWorkerInstance.on('message', async result => {
+                        await commandWorkerInstance.terminate()
+                        commandWorkerInstance = null
+                        resolve(result)
+                    })
+                    commandWorkerInstance.on('error', async error => {
+                        await commandWorkerInstance.terminate()
+                        commandWorkerInstance = null
+                        reject(error)
+                    })
+                    commandWorkerInstance.postMessage(command)
                 }
-                commandWorkerInstance.on('message', result => {
-                    resolve(result)
-                    runNextCommand()
-                })
-                commandWorkerInstance.on('error', error => {
-                    reject(error)
-                    runNextCommand()
-                })
-                commandWorkerInstance.postMessage(command)
-            }
-            start(
-                // resolve()
-                (data) => {
-                    let s
-                    if (!data || (data.code>=300 && data.code<=499)) {
-                        console.error('ERROR: SimplyStore cannot run command ', command.id, data)
-                        if (!data?.code) {
-                            s = {code: 500, status: "failed"}
+                start(
+                    // resolve()
+                    (data) => {
+                        let s
+                        if (!data || (data.code>=300 && data.code<=499)) {
+                            console.error('ERROR: SimplyStore cannot run command ', command.id, data)
+                            if (!data?.code) {
+                                s = {code: 500, status: "failed"}
+                            } else {
+                                s = {code: data.code, status: "failed", message: data.message, details: data.details}
+                            }
+                            status.set(command.id, s)
+                            appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                            mainReject(s)
                         } else {
-                            s = {code: data.code, status: "failed", message: data.message, details: data.details}
+                            s = {code: 200, status: "done"}
+                            status.set(command.id, s)
+                            if (data.data) { // data has changed, commands may do other things instead of changing data
+                                jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
+                                Object.assign(meta, data.meta)
+                                queryWorkerPool.update({
+                                    name: 'update',
+                                    req: {
+                                        body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
+                                        meta
+                                    }
+                                })
+                            }
+                            appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                            mainResolve(s)
                         }
+                    }, 
+                    //reject()
+                    (error) => {
+                        let s = {status: "failed", code: error.code, message: error.message, details: error.details}
                         status.set(command.id, s)
-                    } else {
-                        s = {code: 200, status: "done"}
-                        status.set(command.id, s)
-                        if (data.data) { // data has changed, commands may do other things instead of changing data
-                            jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
-                            Object.assign(meta, data.meta)
-                            queryWorkerPool.update({
-                                name: 'update',
-                                req: {
-                                    body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
-                                    meta
-                                }
-                            })
-                        }
+                        appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                        console.log('command error', command.id, error)
+                        mainReject(s)
                     }
-                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
-                }, 
-                //reject()
-                (error) => {
-                    console.error(error)
-                    let s = {status: "failed", code: error.code, message: error.message, details: error.details}
-                    status.set(command.id, s)
-                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
+                )
+            } else {
+                console.log('no pending commands')
+                // this code can never be triggered from the post(/command/) route, since it always adds a command to the queue
+                // so you can only get here from commandWorkerInstance.on() route
+                // which means that the commandWorkerInstance has finished running the previous command
+                if (commandWorkerInstance) {
+                    await commandWorkerInstance.terminate()
                 }
-            )
-        } else {
-            // this code can never be triggered from the post(/command/) route, since it always adds a command to the queue
-            // so you can only get here from commandWorkerInstance.on() route
-            // which means that the commandWorkerInstance has finished running the previous command
-            await commandWorkerInstance.terminate()
-            commandWorkerInstance.unref() // @FIXME is this needed?
-            commandWorkerInstance = null  // @FIXME or this?
-        }
+                mainResolve(false)
+            }
+        })
     }
 
     function checkCommand(req, res) {
@@ -368,13 +400,12 @@ async function main(options) {
             sendResponse({code:422, body: JSON.stringify(error)}, res)
             return false      
         }
-        appendFile(commandLog, JSONTag.stringify(command))
+        appendFile(commandLog, JSONTag.stringify(command)) //FIXME: this loses request data
         appendFile(commandStatus, JSONTag.stringify(commandOK))
         status.set(command.id, commandOK) 
         sendResponse({code: 202, body: JSON.stringify(commandOK)}, res)
         return command.id
     }
-
 }
 
 function sendResponse(response, res) {
@@ -435,4 +466,4 @@ function handleWebRequest(req,res,options)
     } else {
         res.sendFile('/index.html', fileOptions)
     }
-}
+}