@muze-nl/simplystore 0.5.3 → 0.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.5.3",
+	"version": "0.5.5",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {

package/src/command-worker-module.mjs

@@ -1,5 +1,5 @@
 import JSONTag from '@muze-nl/jsontag'
-import {source} from './symbols.mjs'
+import {source, isChanged} from './symbols.mjs'
 import fastParse from './fastParse.mjs'
 import {stringToSAB,resultSetStringify} from './fastStringify.mjs'
 import writeFileAtomic from 'write-file-atomic'
@@ -25,7 +25,7 @@ export const metaIdProxy = {
         })
     },
     set: (id,ref) => {
-        //FICME: is this correct?
+        //FIXME: is this correct?
         meta.index.id.set(id, resultSet.length-1)
     },
     get: (id) => {
@@ -46,7 +46,10 @@ export const metaIdProxy = {
 export const FastJSONTag = {
     getType: (obj) => JSONTag.getType(obj[source]),
     getAttribute: (obj, attr) => JSONTag.getAttribute(obj[source],attr),
-    setAttribute: (obj, attr, value) => JSONTag.setAttribute(obj[source], attr, value),
+    setAttribute: (obj, attr, value) => {
+        obj[isChanged] = true
+        return JSONTag.setAttribute(obj[source], attr, value)
+    },
     getAttributes: (obj) => JSONTag.getAttributes(obj[source]),
     getAttributeString: (obj) => JSONTag.getAttributesString(obj[source]),
     getTypeString: (obj) => JSONTag.getTypeString(obj[source])
@@ -59,6 +62,7 @@ export async function initialize(task) {
     metaProxy.index.id = metaIdProxy
     datafile = task.datafile
     commands = await import(task.commandsFile).then(mod => {
+        console.log('commands loaded:',Object.keys(mod.default))
         return mod.default
     })
 }
@@ -71,29 +75,29 @@ export default async function runCommand(commandStr, request) {
         jsontag: true
     }
     if (commands[task.name]) {
-        try {
-            commands[task.name](dataspace, task, request, metaProxy)
-            FastJSONTag.setAttribute(dataspace, 'command', task.id)
+        let time = Date.now()
+        commands[task.name](dataspace, task, request, metaProxy)
+        //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
+        FastJSONTag.setAttribute(dataspace, 'command', task.id)
 
-            const strData = resultSetStringify(resultSet)
-            const uint8sab = stringToSAB(strData)
-            response.data = uint8sab
-            response.meta = {
-                index: {
-                    id: meta.index.id
-                }
+        const strData = resultSetStringify(resultSet)
+        const uint8sab = stringToSAB(strData)
+        response.data = uint8sab
+        response.meta = {
+            index: {
+                id: meta.index.id
             }
-
-            await writeFileAtomic(datafile, uint8sab)
-        } catch(err) {
-            console.error('error',err)
-            response.code = 422;
-            response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
         }
+        //TODO: write data every x commands or x minutes, in seperate thread
+        await writeFileAtomic(datafile, uint8sab)
+        let end = Date.now()
+        console.log('task time',end-time)
     } else {
-        console.error('Command not found', task.name, commands)
-        response.code = 404
-        response.body = '<object class="Error">{"message":"Command '+task.name+' not found","code":404}'
+        console.error('Command not found', task.name)
+        throw {
+            code: 404,
+            message: "Command "+task.name+" not found"
+        }
     }
     return response
 }

package/src/command-worker.mjs

@@ -3,11 +3,7 @@ import runCommand, { initialize } from '../src/command-worker-module.mjs'
 
 parentPort.on('message', async data => {
     let result
-    try {
-        await initialize(data)
-        result = await runCommand(data.command)
-    } catch(err) {
-        result = { error: err.message }
-    }
+    await initialize(data)
+    result = await runCommand(data.command)
     parentPort.postMessage(result)
 })

package/src/fastParse.mjs

@@ -838,10 +838,12 @@ export default function parse(input, meta, immutable=true)
             set(target, prop, value) {
                 if (!immutable) {
                     firstParse()
-                    if (JSONTag.getType(value)==='object' && !value[isProxy]) {
-                        value = getNewValueProxy(value)
+                    if (prop!==isChanged) {
+                        if (JSONTag.getType(value)==='object' && !value[isProxy]) {
+                            value = getNewValueProxy(value)
+                        }
+                        target[prop] = value
                     }
-                    target[prop] = value
                     targetIsChanged = true
                     return true
                 }

package/src/server.mjs

@@ -29,43 +29,43 @@ async function main(options) {
     const commandLog    = options.commandLog    || './command-log.jsontag'
     const commandStatus = options.commandStatus || './command-status.jsontag'
 
-	server.use(express.static(wwwroot))
+    server.use(express.static(wwwroot))
 
-	// allow access to raw body, used to parse a query send as post body
+    // allow access to raw body, used to parse a query send as post body
     server.use(express.raw({
         type: (req) => true // parse body on all requests
     }))
 
     function loadData() {
-    	return new Promise((resolve,reject) => {
-	    	let worker = new Worker(loadWorker)
-	    	worker.on('message', result => {
-	    		resolve(result)
-	    		worker.terminate()
-	    	})
-	    	worker.on('error', error => {
-	    		reject(error)
-	    		worker.terminate()
-	    	})
-	    	worker.postMessage(datafile)
-	    })
+        return new Promise((resolve,reject) => {
+            let worker = new Worker(loadWorker)
+            worker.on('message', result => {
+                resolve(result)
+                worker.terminate()
+            })
+            worker.on('error', error => {
+                reject(error)
+                worker.terminate()
+            })
+            worker.postMessage(datafile)
+        })
     }
     try {
         let data = await loadData()
-	    jsontagBuffer = data.data
+        jsontagBuffer = data.data
         meta = data.meta
-	} catch(err) {
-		console.error('ERROR: SimplyStore cannot load '+datafile, err)
-		process.exit(1)
-	}
+    } catch(err) {
+        console.error('ERROR: SimplyStore cannot load '+datafile, err)
+        process.exit(1)
+    }
 
     const queryWorkerInitTask = () => { 
         return {
-        	name: 'init',
-        	req: {
-        		body: jsontagBuffer,
+            name: 'init',
+            req: {
+                body: jsontagBuffer,
                 meta
-        	}
+            }
         }
     }
 
@@ -94,19 +94,19 @@ async function main(options) {
         let path = req.path.substr(6) // cut '/query'
         console.log('query',path)
         let request = {
-        	method: req.method,
-        	url: req.originalUrl,
-        	query: req.query,
-        	path: path
+            method: req.method,
+            url: req.originalUrl,
+            query: req.query,
+            path: path
         }
         if (accept(req,res,['application/jsontag'])) {
             request.jsontag = true
         }
         try {
-        	let result = await queryWorkerPool.run('query', request)
-        	sendResponse(result, res)
+            let result = await queryWorkerPool.run('query', request)
+            sendResponse(result, res)
         } catch(error) {
-        	sendError(error, res)
+            sendError(error, res)
         }
         let end = Date.now()
         console.log(path, (end-start), process.memoryUsage())
@@ -122,20 +122,20 @@ async function main(options) {
         }
         let path = req.path.substr(6) // cut '/query'
         let request = {
-        	method: req.method,
-        	url: req.originalUrl,
-        	query: req.query,
-        	path: path,
-        	body: req.body.toString()
+            method: req.method,
+            url: req.originalUrl,
+            query: req.query,
+            path: path,
+            body: req.body.toString()
         }
         if (accept(req,res,['application/jsontag'])) {
             request.jsontag = true
         }
         try {
-        	let result = await queryWorkerPool.run('query', request)
-        	sendResponse(result, res)
+            let result = await queryWorkerPool.run('query', request)
+            sendResponse(result, res)
         } catch(error) {
-        	sendError(error, res)
+            sendError(error, res)
         }
         let end = Date.now()
         console.log(path, (end-start), process.memoryUsage())
@@ -153,9 +153,13 @@ async function main(options) {
                 let lines = file.split("\n").filter(Boolean) //filter clears empty lines
                 for(let line of lines) {
                     let command = JSONTag.parse(line)
-                    status.set(command.id, command.status)
+                    status.set(command.command, command)
                 }
+            } else {
+                console.error('Could not open command status',commandStatusFile)
             }
+        } else {
+            console.log('no command status', commandStatusFile)
         }
         return status
     }
@@ -208,31 +212,38 @@ async function main(options) {
             start(
                 // resolve()
                 (data) => {
-                    if (!data || data.error) {
-                        console.error('ERROR: SimplyStore cannot run command ', command.id, err)
-                        if (!data.error) {
-                            status.set(command.id, 'failed')
-                            throw new Error('Unexpected command failure')
+                    let s
+                    if (!data || (data.code>=300 && data.code<=499)) {
+                        console.error('ERROR: SimplyStore cannot run command ', command.id, data)
+                        if (!data?.code) {
+                            s = {code: 500, status: "failed"}
                         } else {
-                            status.set(command.id, data.error)
-                            throw data.error
+                            s = {code: data.code, status: "failed", message: data.message, details: data.details}
+                        }
+                        status.set(command.id, s)
+                    } else {
+                        s = {code: 200, status: "done"}
+                        status.set(command.id, s)
+                        if (data.data) { // data has changed, commands may do other things instead of changing data
+                            jsontagBuffer = data.data
+                            meta = data.meta
+                            // restart query workers with new data
+                            let oldPool = queryWorkerPool
+                            queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
+                            setTimeout(() => {
+                                oldPool.close()
+                            }, 2000)
                         }
                     }
-                    jsontagBuffer = data.data
-                    meta = data.meta
-                    status.set(command.id, 'done')
-                    appendFile(commandStatus, JSONTag.stringify({command:command.id, status: 'done'}))
-                    // restart query workers with new data
-                    let oldPool = queryWorkerPool
-                    queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
-                    setTimeout(() => {
-                        oldPool.close()
-                    }, 2000)
+                    let l = Object.assign({command:command.id}, s)
+                    console.log('appending status ',l,s)
+                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                 }, 
                 //reject()
                 (error) => {
-                    status.set(command.id, error)
-                    appendFile(commandStatus, JSONTag.stringify({command:command.id, status: error}))
+                    let s = {status: "failed", code: error.code, message: error.message, details: error.details}
+                    status.set(command.id, s)
+                    appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                 }
             )
         } else {
@@ -270,8 +281,9 @@ async function main(options) {
 
             runNextCommand()
         } catch(err) {
-            appendFile(commandStatus, JSONTag.stringify({command:commandId, status: 'ERROR: '+err.message}))
-            status.set(commandId, 'ERROR: '+err.message)
+            let s = {code:err.code||500, status:'failed', message:err.message, details:err.details}
+            status.set(commandId, s)
+            appendFile(commandStatus, JSONTag.stringify(Object.assign({command:commandId}, s)))
             console.error('ERROR: SimplyStore cannot run command ', commandId, err)
         }
     })
@@ -279,32 +291,35 @@ async function main(options) {
     function checkCommand(req, res) {
         let commandStr = req.body.toString() // raw body through express.raw()
         let command = JSONTag.parse(commandStr)
+        let commandOK = {
+            command: command?.id,
+            code: 202,
+            status: 'accepted'
+        }
         if (!command || !command.id) {
             error = {
                 code: 422,
-                message: "Command has no id"
+                message: "Command has no id",
+                details: command
             }
             sendResponse({code: 422, body: JSON.stringify(error)}, res)
             return false
         } else if (status.has(command.id)) {
-            result = "OK"
-            sendResponse({body: JSON.stringify(result)}, res)
+            sendResponse({body: JSON.stringify(s)}, res)
             return false
         } else if (!command.name) {
             error = {
                 code: 422,
-                message: "Command has no name"
+                message: "Command has no name",
+                details: command
             }
             sendResponse({code:422, body: JSON.stringify(error)}, res)
             return false      
         }
         appendFile(commandLog, JSONTag.stringify(command))
-        appendFile(commandStatus, JSONTag.stringify({
-            command: command.id,
-            status: 'accepted'
-        }))
-        status.set(command.id, 'accepted') 
-        sendResponse({code: 202, body: '"Accepted"'}, res)
+        appendFile(commandStatus, JSONTag.stringify(commandOK))
+        status.set(command.id, commandOK) 
+        sendResponse({code: 202, body: JSON.stringify(commandOK)}, res)
         return command.id
     }
 
@@ -319,7 +334,7 @@ async function main(options) {
             sendResponse({
                 code: 404,
                 jsontag: false,
-                body: JSON.stringify({code: 404, message: "Command not found"})
+                body: JSON.stringify({code: 404, message: "Command not found", details: req.params.id})
             }, res)
         }
     })