@muze-nl/simplystore 0.3.2 → 0.3.5

package/src/server.mjs

@@ -1,86 +1,46 @@
 import express from 'express'
 import fs from 'fs'
-import pointer from 'json-pointer'
 import JSONTag from '@muze-nl/jsontag'
-import {JSONPath} from 'jsonpath-plus'
-import {VM} from 'vm2'
-import {_,from,not,anyOf,allOf} from 'array-where-select'
 import { fileURLToPath } from 'url'
 import path from 'path'
+import commands from './commands.mjs'
+import {appendFile} from './util.mjs'
+import {Piscina} from 'piscina'
 
 const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url)))
 
 const server    = express()
 
-function deepFreeze(obj) {
-		Object.freeze(obj)
-		Object.keys(obj).forEach(prop => {
-				if (typeof obj[prop] === 'object' && !Object.isFrozen(obj[prop])) {
-						deepFreeze(obj[prop])
-				}
-		})
-		return obj
-}
-
-function isString(s)
-{
-    return typeof s === 'string' || s instanceof String
-}
-
-function joinArgs(args) {
-    return args = args.map(arg => {
-        if (isString(arg)) {
-            return arg
-        } else {
-            return JSONTag.stringify(arg)
-        }
-    }).join(' ')
-}
-
-function connectConsole(res) {
-    return {
-        log: function(...args) {
-            res.append('X-Console-Log', joinArgs(args))
-        },
-        warning: function(...args) {
-            res.append('X-Console-Warning', joinArgs(args))
-        },
-        error: function(...args) {
-            res.append('X-Console-Error', joinArgs(args))            
-        }
-    }
-}
-
 async function main(options) {
     if (!options) {
         options = {}
     }  
-	  const port     = options.port || 3000
-	  const datafile = options.datafile || 'data.jsontag'
-	  const wwwroot  = options.wwwroot || __dirname+'/www'
-	  let meta       = options.meta || {}
-	  let dataspace  = options.dataspace || null
+    const port          = options.port          || 3000
+    const datafile      = options.datafile      || './data.jsontag'
+    const wwwroot       = options.wwwroot       || __dirname+'/www'
+    const commandLog    = options.commandlog    || './commandlog.jsontag'
+    const queryWorker   = options.queryWorker   || __dirname+'/src/worker-query-init.mjs'
+    const commandWorker = options.commandWorker || __dirname+'/src/worker-command-init.mjs'
 
-    const originalJSON = JSON
-    JSON = JSONTag // monkeypatching
+    let jsontag         = fs.readFileSync(datafile, 'utf-8')
 
-    if (!dataspace) {
-        console.log('loading data...')
-        let file = fs.readFileSync(datafile)
-        try {
-        dataspace = JSONTag.parse(file.toString(), null, meta)
-        } catch(e) {
-            console.error(e)
-            process.exit()
+    function initWorkerPool(workerName, size=null) {
+        let options = {
+            filename: workerName,
+            workerData: jsontag
+        }
+        if (size) {
+            options.maxThreads = size
         }
+        return new Piscina(options)
     }
-    deepFreeze(dataspace)
 
-    let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-    console.log(`data loaded (${used} MB)`);
+    let queryWorkerpool = initWorkerPool(queryWorker)
+
+//    const commandWorkerpool = initWorkerPool(commandWorker,1) // only one update worker so no changes can get lost
 
     server.get('/', (req,res) => {
-        res.send('<h1>SimplyStore</h1>') //TODO: implement something nice
+        res.send('<h1>SimplyStore</h1>') //@TODO: implement something nice
     })
 
     server.use(express.static(wwwroot))
@@ -103,35 +63,34 @@ async function main(options) {
         return true
     }
 
-    function getDataSpace(req, res, dataspace) {
-        let path = req.path.substr(6); // cut '/query'
-        if (!path) {
-            path = '';
+    function sendResponse(response, res) {
+        if (response.code) {
+            res.status(response.code)
         }
-        if (path.substring(path.length-1)==='/') {
-            //jsonpointer doesn't allow a trailing '/'
-            path = path.substring(0, path.length-1)
+        if (response.jsontag) {
+            res.setHeader('content-type','application/jsontag')
+        } else {
+            res.setHeader('content-type','application/json')
         }
-        let result
-        if (path) {
-            //jsonpointer doesn't allow an empty pointer
-            try {
-                if (pointer.has(dataspace, path)) {
-                    result = pointer.get(dataspace, path)
-                } else {
-                    result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
-                }
-            } catch(err) {
-                result = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
-            }
+        res.send(response.body)+"\n"
+    }
+
+    function sendCommandResponse(result, req, res) {
+        if (result.code) {
+            res.status(result.code)
+        }
+        if (req.accepts('application/jsontag')) {
+            res.setHeader('content-type','application/jsontag')
+            res.send(JSONTag.stringify(result, null, 4)+"\n")
         } else {
-            result = dataspace
+            res.setHeader('content-type','application/json')
+            res.send(JSON.stringify(result, null, 4)+"\n")
         }
-        return [result,path]
     }
-    
+
     server.get('/query/*', (req, res, next) => 
     {
+        console.log('express query')
         let start = Date.now()
 
         if ( !accept(req,res,
@@ -151,21 +110,19 @@ async function main(options) {
             // done
             return
         }
-
-        let [result,path] = getDataSpace(req, res, dataspace)
-        if (JSONTag.getAttribute(result, 'class')==='Error') {
-            res.status(result.code)
-        }
-        result = linkReplacer(result, path+'/')
-        if (req.accepts('application/jsontag')) {
-            res.setHeader('content-type','application/jsontag+json')
-            res.send(JSONTag.stringify(result, null, 4)+"\n")
-        } else {
-            res.setHeader('content-type','application/json')
-            res.send(originalJSON.stringify(result, null, 4)+"\n")
+        let path = req.path.substr(6); // cut '/query'
+        let request = {
+            method: req.method,
+            url: req.originalUrl,
+            query: req.query,
+            jsontag: req.accepts('application/jsontag')
         }
-        let end = Date.now()
-        console.log(path, (end-start), process.memoryUsage())
+        queryWorkerpool.run({pointer:path, request})
+        .then(response => {
+            sendResponse(response, res)
+            let end = Date.now()
+            console.log(path, (end-start), process.memoryUsage())
+        })
     })
 
     /**
@@ -178,62 +135,111 @@ async function main(options) {
         ) {
             return
         }
-        let error
-        let [result,path] = getDataSpace(req, res, dataspace)
-        if (result) {
-            // do the query here
-            let query = req.body.toString() // raw body through express.raw()
-            // @todo add text search: https://github.com/nextapps-de/flexsearch
-            // @todo add tree walk map/reduce/find/filter style functions
-            // @todo add arc tree dive function?
-            const vm = new VM({
-//                timeout: 1000,
-                allowAsync: false,
-                sandbox: {
-                    root: dataspace,
-                    data: result,
-                    meta: meta,
-                    _: _,
-                    from: from,
-                    not: not,
-                    anyOf: anyOf,
-                    allOf: allOf,
-                    console: connectConsole(res),
-                    JSONTag: JSONTag,
-                    request: {
-                        method: req.method,
-                        url: req.originalUrl,
-                        query: req.query
-                    }
-                },
-                wasm: false
-            })
-            try {
-                result = vm.run(query)
-                let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-                console.log(`(${used} MB)`);
-            } catch(err) {
-                console.log(err)
-                error = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(''+err)+',"code":422}')
-            }
+        let query = req.body.toString() // raw body through express.raw()
+        let path = req.path.substr(6); // cut '/query'
+        let request = {
+            method: req.method,
+            url: req.originalUrl,
+            query: req.query,
+            jsontag: req.accepts('application/jsontag')
         }
+        queryWorkerpool.run({pointer:path, request, query})
+        .then(response => {
+            sendResponse(response, res)
+            let end = Date.now()
+            console.log(path, (end-start), process.memoryUsage())
+        })
+    })
 
-        if (error) {
-            res.status(error.code)
-            result = error
-        }
-        if (req.accepts('application/jsontag')) {
-            res.setHeader('content-type','application/jsontag+json')
-            res.send(JSONTag.stringify(result, null, 4)+"\n")
+    let status = new Map()
+
+    server.get('/command/:id', (req, res) => {
+        //@TODO: find the status of command with :id
+        //return that
+        if (status.has(req.params.id)) {
+            let result = status.get(req.params.id)
+            sendResponse({
+                jsontag: false,
+                body: JSON.stringify(result)
+            },res)
         } else {
-            res.setHeader('content-type','application/json')
-            res.send(originalJSON.stringify(result, null, 4)+"\n")
+            sendResponse({
+                code: 404,
+                jsontag: false,
+                body: JSON.stringify({"code":404,"message":"Command not found"})
+            }, res)
         }
-        let end = Date.now()
-        console.log(path, (end-start), process.memoryUsage())
     })
 
+    server.post('/command', async (req, res) => {
+        let start = Date.now()
+        if ( !accept(req,res,
+            ['application/jsontag','application/json']) 
+        ) {
+            return
+        }
+        let error, result
+
+        let commandStr = req.body.toString() // raw body through express.raw()
+        let command = JSONTag.parse(commandStr)
+        if (!command.id) {
+            error = {
+                code: 422,
+                message: "Command has no id"
+            }
+            sendCommandResponse(error, req, res)
+            return
+        } else if (status.has(command.id)) {
+            result = "OK"
+            sendCommandResponse(result, req, res)
+            return
+        } else if (!command.name || !commands[command.name]) {
+            error = {
+                code: 422,
+                message: "Command has no name or is unknown"
+            }
+            sendCommandResponse(error, req, res)
+            return            
+        }
+        await appendFile(commandLog, JSONTag.stringify(command))
+
+        status.set(command.id, 'queued')
+        console.log('command',command)
+
+        result = "OK"
+        sendCommandResponse(result, req, res)
+        let request = {
+            method: req.method,
+            url: req.originalUrl,
+            query: req.query,
+            jsontag: req.accepts('application/jsontag')
+        }
+
+        commandWorkerpool
+        .run({request, commandStr})
+        .then(response => {
+            //@TODO store response status, if response.code => error
+            if (!response.code) {
+                jsontag = response.body // global jsontag
+                let dataspace = JSONTag.parse(jsontag)
+                //@TODO: make sure queryWorkerpool is only replaced after
+                //workers are initialized, to prevent hickups if initialization takes a long time
+                let newQueryWorkerpool = initWorkerPool('./worker-query')
+                queryWorkerpool.terminate() // gracefully
+                queryWorkerpool = newQueryWorkerpool
+                //@TODO: write dataspace to disk
+                status.set(command.id, 'done')
+                let end = Date.now()
+                console.log(command.name, (end-start), process.memoryUsage())        
+            }
+        })
+        .catch(err => {
+            console.error(err)
+            //@TODO: set status for this command to error with this err
+            status.set(command.id, err)
+        })
 
+    })
 
     function handleWebRequest(req,res,options)
     {

package/src/share.mjs

@@ -0,0 +1,159 @@
+import JSONTag from '@muze-nl/jsontag'
+
+const handler = function(root, index, buffer) 
+{
+	return {
+		get(target, key, receiver) 
+		{
+			//@FIXME: array.length needs to be handled
+			//@FIXME: array functions need to be handled
+			let keyNumber = index.key[key]
+			if (root[keyNumber]) {
+				let keyStart = root[keyNumber].s + root.s
+				let keyEnd = root[keyNumber].e + root.s
+				// subobjecten hoeven niet geparsed...
+				if (root[keyNumber].c) {
+					// object
+					return new jsontagProxy(root[keyNumber], index, buffer)
+				}
+				return JSON.parse(buffer.slice(keyStart,keyEnd))
+			}
+		},
+
+		set() {
+			throw new Error('This data is immutable')
+		},
+
+		deleteProperty() {
+			throw new Error('This data is immutable')
+		},
+
+		has(target, key)
+		{
+			let keyNumber = index.key[key]
+			return typeof root[keyNumber] !== 'undefined'
+		},
+
+		ownKeys(target)
+		{
+			return root.c.map(n => index.reverse[n])
+		}
+	}
+}
+
+export function getJsontagProxy(root, index, buffer) {
+	return new Proxy({}, handler(root, index, buffer))
+}
+
+export function share(data)
+{
+	let jsontag = JSONTag.stringify(data)
+	let buffer = new SharedArrayBuffer(jsontag.length)
+	let dv = new DataView(buffer)
+	let encoder = new TextEncoder('utf-8')
+	let root = {}
+	let index = {
+		key: {},
+		reverse: [],
+		types: {}
+	}
+	let current = 0
+	let seen = new Map()
+
+	function getKey(key) {
+		let l;
+		if (typeof index.key[key] == 'undefined') {
+			l = Object.keys(index.key).length
+			index.key[key] = l
+			index.reverse[l] = key
+		}
+		return index.key[key]
+	}
+
+	function getType(type) {
+		if (typeof index.types[type] == 'undefined') {
+			let l = Object.keys(index.types).length;
+			index.types[type] = l
+		}
+
+	}
+
+	function store(container, key, value) {
+		let l = getKey(key)
+		if (seen.has(value)) {
+			container[l] = seen.get(value)	
+			return
+		}
+		container[l] = {
+			s: current
+		}
+		container = container[l]
+		let attributes = JSONTag.getAttributes(value)
+		if (attributes) {
+			container.a = {
+				s: current,
+				c: {}
+			}
+			Object.entries(attributes).forEach(([k,v]) => {
+				let l = getKey(k)
+				let t = getType('string')
+				container.a.c[k] = {
+					s: current,
+					t
+				}
+				v = JSON.stringify(v)
+				let {r,w} = encoder.encodeInto(v, dv, current)
+				current += w
+				container.a.c[k].e = current
+			})
+		}
+		let type = JSONTag.getType(value)
+		let t = getType(type)
+		container.t = t
+		let v = JSON.stringify(value)
+		let uint8buffer = new Uint8Array(buffer)
+		let {r,w} = encoder.encodeInto(v, uint8buffer, current)
+		current+= w
+		container.e = current
+	}
+
+
+	function walk(node, parentKey, parent, save) 
+	{
+		if (seen.has(node)) {
+			Object.assign(save, seen.get(node))
+			return
+		}
+		let type = JSONTag.getType(node)
+		let t = getType(type)
+		switch (type) {
+			case 'array':
+				seen.set(node,save)
+				save.s = current
+				save.t = t
+				save.c = []
+				node.forEach((v,k) => walk(v, k, node, save.c))
+				save.e = current
+				break
+			case 'object':
+				seen.set(node, save)
+				save.s = current
+				save.c = []
+				save.t = t
+				Object.entries(node).forEach(([k,v]) => walk(v, k, node, save.c))
+				save.e = current
+				break
+			default:
+				store(save, parentKey, node)
+				break
+		}
+	}
+
+	walk(data, null, null, root)
+	console.log(root)
+	return {
+		root: getJsontagProxy(root, index, buffer),
+		index,
+		buffer
+	}
+}

package/src/util.mjs

@@ -0,0 +1,46 @@
+import JSONTag from '@muze-nl/jsontag'
+import fs from 'node:fs/promises'
+
+export function deepFreeze(obj) {
+		Object.freeze(obj)
+		Object.keys(obj).forEach(prop => {
+				if (typeof obj[prop] === 'object' && !Object.isFrozen(obj[prop])) {
+						deepFreeze(obj[prop])
+				}
+		})
+		return obj
+}
+
+export function isString(s)
+{
+    return typeof s === 'string' || s instanceof String
+}
+
+export function joinArgs(args) {
+    return args = args.map(arg => {
+        if (isString(arg)) {
+            return arg
+        } else {
+            return JSONTag.stringify(arg)
+        }
+    }).join(' ')
+}
+
+/**
+ * atomic append to a log file, adds newline after each write
+ * @param  {string} filename The filename to append to
+ * @param  {string} data     The line to write
+ * @return {void}
+ */
+export async function appendFile(filename, data) {
+	console.log('appending command')
+	let handle;
+	try {
+		handle = await fs.open(filename, 'a')
+		await handle.appendFile(data+"\n")
+		await handle.datasync()
+		return true
+	} finally {
+		await handle.close()
+	}
+}

package/src/worker-command-init.mjs

@@ -0,0 +1,11 @@
+import * as commandWorker from './worker-command.mjs'
+import worker_threads from 'node:worker_threads' 
+
+async function initialize() {
+	let meta = {}
+	let dataspace = JSONTag.parse(worker_threads.workerData, null, meta)
+	await commandWorker.initialize(worker_threads.workerData)
+	return commandWorker.runCommand
+}
+
+export default initialize()

package/src/worker-command.mjs

@@ -0,0 +1,51 @@
+import JSONTag from '@muze-nl/jsontag'
+import commands from './commands.mjs'
+
+/**
+ * Command Worker for threads.js library
+ * returns JSONTag strings, since otherwise JSON.stringify is used
+ * and type+attribute data gets lost
+ */
+
+let dataspace
+
+export function setDataspace(d) {
+	dataspace = d
+}
+
+/**
+ * @TODO: check for valid command id
+ * @TODO: write valid commands to command log, emit 'ok', check in server.mjs for that
+ * @TODO: setTimeout or do above checks in server.mjs before queueing
+ */
+
+export function async initialize(jsontag) {
+    if (!jsontag) { throw new Error('missing jsontag parameter')}
+	dataspace = jsontag
+	console.log('initialized command worker thread')
+	return true
+}
+
+export function runCommand({request, commandStr}) {
+    if (!commandStr) { throw new Error('missing command parameter')}
+    if (!request) { throw new Error('missing request parameter')}
+	let response = {
+		jsontag: true
+	}
+    let command = JSONTag.parse(commandStr) // raw body through express.raw()
+    if (command && command.name && commands[command.name]) {
+        try {
+        	commands[command.name](dataspace, command, request)
+            response.body = JSONTag.stringify(dataspace) //@TODO: this is inefficient, patch would be better
+        } catch(err) {
+			console.log(err)
+            response.code = 422;
+           	response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
+        }
+    } else {
+    	response.code = 404
+       	response.body = '<object class="Error">{"message":"Command '+command.name+' not found","code":404}'
+    }
+    return response
+}
+

package/src/worker-query-init.mjs

@@ -0,0 +1,14 @@
+import JSONTag from '@muze-nl/jsontag';
+import worker_threads from 'node:worker_threads';
+import * as queryWorker from './worker-query.mjs';
+
+async function initialize() {
+	let meta = {}
+	let dataspace = JSONTag.parse(worker_threads.workerData, null, meta)
+	//console.log('starting')
+	await queryWorker.initialize(dataspace,meta)
+	//console.log('initialized')
+	return queryWorker.runQuery
+}
+
+export default initialize()