@muze-nl/simplystore 0.3.3 → 0.3.6

package/.eslintrc.cjs

@@ -0,0 +1,15 @@
+module.exports = {
+    "env": {
+        "browser": true,
+        "es2021": true
+    },
+    "extends": "eslint:recommended",
+    "overrides": [
+    ],
+    "parserOptions": {
+        "ecmaVersion": "latest",
+        "sourceType": "module"
+    },
+    "rules": {
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.3.3",
+	"version": "0.3.6",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -22,10 +22,11 @@
 		"express": "^4.18.1",
 		"json-pointer": "^0.6.2",
 		"jsonpath-plus": "^7.2.0",
-		"threads": "^1.7.0",
+		"piscina": "^4.1.0",
 		"vm2": "^3.9.13"
 	},
 	"devDependencies": {
+		"eslint": "^8.48.0",
 		"process": "^0.11.10",
 		"tap": "^16.3.8"
 	}

package/src/commands.mjs

@@ -1,5 +1,6 @@
 export default {
 	addPerson: (dataspace, command) => {
+		console.log('dataspace',dataspace)
 		dataspace.persons.push(command.value)
 //		dataspace.persons[0].name = 'Jan';
 		return 'foo'

package/src/server.mjs

@@ -3,9 +3,9 @@ import fs from 'fs'
 import JSONTag from '@muze-nl/jsontag'
 import { fileURLToPath } from 'url'
 import path from 'path'
-import { spawn, Pool, Worker } from 'threads'
 import commands from './commands.mjs'
 import {appendFile} from './util.mjs'
+import {Piscina} from 'piscina'
 
 const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url)))
 
@@ -15,29 +15,32 @@ async function main(options) {
     if (!options) {
         options = {}
     }  
-    const port       = options.port     || 3000
-    const datafile   = options.datafile || 'data.jsontag'
-    const wwwroot    = options.wwwroot  || __dirname+'/www'
-    const commandLog = options.commandlog || 'commandlog.jsontag'
-    let meta         = options.meta     || {}
+    const port          = options.port          || 3000
+    const datafile      = options.datafile      || './data.jsontag'
+    const wwwroot       = options.wwwroot       || __dirname+'/www'
+    const commandLog    = options.commandlog    || './commandlog.jsontag'
+    const queryWorker   = options.queryWorker   || __dirname+'/src/worker-query-init.mjs'
+    const commandWorker = options.commandWorker || __dirname+'/src/worker-command-init.mjs'
 
-    let jsontag      = fs.readFileSync(datafile, 'utf-8')
+    let jsontag         = fs.readFileSync(datafile, 'utf-8')
 
     function initWorkerPool(workerName, size=null) {
-        return Pool(() => {
-            return spawn(new Worker(workerName)).then(worker => {
-                worker.initialize(jsontag)
-                return worker
-            })
-        }, size)
+        let options = {
+            filename: workerName,
+            workerData: jsontag
+        }
+        if (size) {
+            options.maxThreads = size
+        }
+        return new Piscina(options)
     }
 
-    let queryWorkerpool = initWorkerPool('./worker-query')
+    let queryWorkerpool = initWorkerPool(queryWorker)
 
-    const commandWorkerpool = initWorkerPool('./worker-command',1) // only one update worker so no changes can get lost
+    const commandWorkerpool = initWorkerPool(commandWorker,1) // only one update worker so no changes can get lost
 
     server.get('/', (req,res) => {
-        res.send('<h1>SimplyStore</h1>') //TODO: implement something nice
+        res.send('<h1>SimplyStore</h1>') //@TODO: implement something nice
     })
 
     server.use(express.static(wwwroot))
@@ -114,9 +117,8 @@ async function main(options) {
             query: req.query,
             jsontag: req.accepts('application/jsontag')
         }
-        queryWorkerpool.queue(queryWorker => {
-            return queryWorker.runQuery(path, request)
-        }).then(response => {
+        queryWorkerpool.run({pointer:path, request})
+        .then(response => {
             sendResponse(response, res)
             let end = Date.now()
             console.log(path, (end-start), process.memoryUsage())
@@ -141,9 +143,8 @@ async function main(options) {
             query: req.query,
             jsontag: req.accepts('application/jsontag')
         }
-        queryWorkerpool.queue(queryWorker => {
-            return queryWorker.runQuery(path, request, query)
-        }).then(response => {
+        queryWorkerpool.run({pointer:path, request, query})
+        .then(response => {
             sendResponse(response, res)
             let end = Date.now()
             console.log(path, (end-start), process.memoryUsage())
@@ -215,7 +216,7 @@ async function main(options) {
         }
 
         commandWorkerpool
-        .queue(commandWorker => commandWorker.runCommand(request, commandStr))
+        .run({request, commandStr})
         .then(response => {
             //@TODO store response status, if response.code => error
             if (!response.code) {
@@ -223,8 +224,8 @@ async function main(options) {
                 let dataspace = JSONTag.parse(jsontag)
                 //@TODO: make sure queryWorkerpool is only replaced after
                 //workers are initialized, to prevent hickups if initialization takes a long time
-                let newQueryWorkerpool = initWorkerPool('./worker-query')
-                queryWorkerpool.terminate() // gracefully
+                let newQueryWorkerpool = initWorkerPool(queryWorker)
+                queryWorkerpool.destroy() // gracefully
                 queryWorkerpool = newQueryWorkerpool
                 //@TODO: write dataspace to disk
                 status.set(command.id, 'done')

package/src/worker-command-init.mjs

@@ -0,0 +1,12 @@
+import * as commandWorker from './worker-command.mjs'
+import worker_threads from 'node:worker_threads' 
+import JSONTag from '@muze-nl/jsontag'
+
+async function initialize() {
+	let meta = {}
+	let dataspace = JSONTag.parse(worker_threads.workerData, null, meta)
+	await commandWorker.initialize(dataspace,meta)
+	return commandWorker.runCommand
+}
+
+export default initialize()

package/src/worker-command.mjs

@@ -0,0 +1,51 @@
+import JSONTag from '@muze-nl/jsontag'
+import commands from './commands.mjs'
+
+/**
+ * Command Worker for threads.js library
+ * returns JSONTag strings, since otherwise JSON.stringify is used
+ * and type+attribute data gets lost
+ */
+
+let dataspace, meta
+
+export function setDataspace(d) {
+	dataspace = d
+}
+
+/**
+ * @TODO: check for valid command id
+ * @TODO: write valid commands to command log, emit 'ok', check in server.mjs for that
+ * @TODO: setTimeout or do above checks in server.mjs before queueing
+ */
+
+export async function initialize(jsontag, m) {
+    if (!jsontag) { throw new Error('missing jsontag parameter')}
+	dataspace = jsontag
+	meta = m
+	return true
+}
+
+export function runCommand({request, commandStr}) {
+    if (!commandStr) { throw new Error('missing command parameter')}
+    if (!request) { throw new Error('missing request parameter')}
+	let response = {
+		jsontag: true
+	}
+    let command = JSONTag.parse(commandStr) // raw body through express.raw()
+    if (command && command.name && commands[command.name]) {
+        try {
+        	commands[command.name](dataspace, command, request)
+            response.body = JSONTag.stringify(dataspace) //@TODO: this is inefficient, patch would be better
+        } catch(err) {
+			console.log(err)
+            response.code = 422;
+           	response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
+        }
+    } else {
+    	response.code = 404
+       	response.body = '<object class="Error">{"message":"Command '+command.name+' not found","code":404}'
+    }
+    return response
+}
+

package/src/worker-query-init.mjs

@@ -0,0 +1,14 @@
+import JSONTag from '@muze-nl/jsontag';
+import worker_threads from 'node:worker_threads';
+import * as queryWorker from './worker-query.mjs';
+
+async function initialize() {
+	let meta = {}
+	let dataspace = JSONTag.parse(worker_threads.workerData, null, meta)
+	//console.log('starting')
+	await queryWorker.initialize(dataspace,meta)
+	//console.log('initialized')
+	return queryWorker.runQuery
+}
+
+export default initialize()

package/src/worker-query.mjs

@@ -0,0 +1,146 @@
+import JSONTag from "@muze-nl/jsontag"
+import pointer from 'json-pointer'
+import {_,from,not,anyOf,allOf} from 'array-where-select'
+import {deepFreeze} from './util.mjs'
+import {VM} from 'vm2'
+
+let dataspace, meta = {};
+
+export function setDataspace(d, m) {
+    dataspace = d
+    if(m) {
+        meta = m
+    }
+}
+
+export function getDataSpace(path, dataspace) {
+    if (path.substring(path.length-1)==='/') {
+        //jsonpointer doesn't allow a trailing '/'
+        path = path.substring(0, path.length-1)
+    }
+    let result
+    if (path) {
+        //jsonpointer doesn't allow an empty pointer
+        try {
+            if (pointer.has(dataspace, path)) {
+                result = pointer.get(dataspace, path)
+            } else {
+                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+            }
+        } catch(err) {
+            result = JSONTag.parse('<object class="Error">{"message":'+JSON.stringify(err.message)+', "code":500}')
+        }
+    } else {
+        result = dataspace
+    }
+    return [result,path]
+}
+
+export function linkReplacer(data, baseURL) {
+    let type = JSONTag.getType(data)
+    let attributes = JSONTag.getAttributes(data)
+    if (Array.isArray(data)) {
+        data = data.map((entry,index) => {
+            return linkReplacer(data[index], baseURL+index+'/')
+        })
+    } else if (type === 'link') {
+        // do nothing
+    } else if (data && typeof data === 'object') {
+        data = JSONTag.clone(data)
+        Object.keys(data).forEach(key => {
+            if (Array.isArray(data[key])) {
+                data[key] = new JSONTag.Link(baseURL+key+'/')
+            } else if (data[key] && typeof data[key] === 'object') {
+                if (JSONTag.getType(data[key])!=='link') {
+                    let id=JSONTag.getAttribute(data[key], 'id')
+                    if (!id) {
+                        id = baseURL+key+'/'
+                    }
+                    data[key] = new JSONTag.Link(id)
+                }
+            }
+        })
+    }
+    return data
+}
+
+//@TODO: emit console events that server.mjs picks up
+function connectConsole(res) {
+    return {
+        log: function(...args) {
+//            res.append('X-Console-Log', joinArgs(args))
+        },
+        warning: function(...args) {
+//            res.append('X-Console-Warning', joinArgs(args))
+        },
+        error: function(...args) {
+//            res.append('X-Console-Error', joinArgs(args))            
+        }
+    }
+}
+
+export async function initialize(jsontag, m) {
+    if (!jsontag) { throw new Error('missing jsontag parameter')}
+	dataspace = jsontag
+    meta = m
+    deepFreeze(dataspace)
+    return true
+}
+
+export function runQuery({pointer, request, query}) {
+    if (!pointer) { throw new Error('missing pointer parameter')}
+    if (!request) { throw new Error('missing request parameter')}
+	console.log('query',pointer)
+	let response = {
+        jsontag: request.jsontag
+    }
+    let [result,path] = getDataSpace(pointer, dataspace)
+
+    if (query) {
+        // @todo add text search: https://github.com/nextapps-de/flexsearch
+        // @todo add tree walk map/reduce/find/filter style functions
+        // @todo add arc tree dive function?
+        // @todo replace VM with V8 isolate
+        const vm = new VM({
+            timeout: 1000,
+            allowAsync: false,
+            sandbox: {
+                root: dataspace, //@TODO: if we don't pass the root, we can later shard
+                data: result,
+                meta,
+                _,
+                from,
+                not,
+                anyOf,
+                allOf,
+//                    console: connectConsole(res),
+                JSONTag,
+                request
+            },
+            wasm: false
+        })
+        try {
+            result = vm.run(query)
+            let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+            console.log(`(${used} MB)`);
+        } catch(err) {
+            console.log(err)
+            response.code = 422;
+            if (request.jsontag) {
+            	response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
+            } else {
+            	response.body = JSON.stringify({message:err, code: 422})
+            }
+        }
+    } else {
+        result = linkReplacer(result, path+'/')
+    }
+    if (!response.code) {
+        if (response.jsontag) {
+        	response.body = JSONTag.stringify(result)
+        } else {
+        	response.body = JSON.stringify(result)
+        }
+    }
+    return response
+}

package/.npmignore~

@@ -1,8 +0,0 @@
-node_modules
-.git
-.gitmodules
-tests
-example
-design
-docs
-test-command.jsontag

package/package.json~

@@ -1,32 +0,0 @@
-{
-	"name": "@muze-nl/simplystore",
-	"version": "0.3.2",
-	"main": "src/server.mjs",
-	"type": "module",
-	"scripts": {
-		"start": "node src/run.mjs",
-		"test": "tap test/*.mjs"
-	},
-	"author": "auke@muze.nl",
-	"license": "MIT",
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/simplyedit/simplystore.git"
-	},
-	"bugs": "https://github.com/simplyedit/simplystore/issues",
-	"homepage": "https://github.com/simplyedit/simplystore#readme",
-	"dependencies": {
-		"@muze-nl/jsontag": "^0.8.5",
-		"array-where-select": "^0.3.1",
-		"codemirror": "^6.0.1",
-		"express": "^4.18.1",
-		"json-pointer": "^0.6.2",
-		"jsonpath-plus": "^7.2.0",
-		"threads": "^1.7.0",
-		"vm2": "^3.9.13"
-	},
-	"devDependencies": {
-		"process": "^0.11.10",
-		"tap": "^16.3.8"
-	}
-}

package/src/worker-command.js

@@ -1,49 +0,0 @@
-import {expose} from 'threads/worker'
-import JSONTag from '@muze-nl/jsontag'
-import commands from './commands.mjs'
-
-/**
- * Command Worker for threads.js library
- * returns JSONTag strings, since otherwise JSON.stringify is used
- * and type+attribute data gets lost
- */
-
-let dataspace
-
-/**
- * @TODO: check for valid command id
- * @TODO: write valid commands to command log, emit 'ok', check in server.mjs for that
- * @TODO: setTimeout or do above checks in server.mjs before queueing
- */
-const command = {
-	initialize(jsontag) {
-        if (!jsontag) { throw new Error('missing jsontag parameter')}
-		dataspace = JSONTag.parse(jsontag)
-		console.log('initialized command worker thread')
-		return true
-	},
-	runCommand(request, commandStr) {
-        if (!commandStr) { throw new Error('missing command parameter')}
-        if (!request) { throw new Error('missing request parameter')}
-		let response = {
-			jsontag: true
-		}
-        let command = JSONTag.parse(commandStr) // raw body through express.raw()
-        if (command && command.name && commands[command.name]) {
-            try {
-            	commands[command.name](dataspace, command, request)
-                response.body = JSONTag.stringify(dataspace) //@TODO: this is inefficient, patch would be better
-            } catch(err) {
-				console.log(err)
-                response.code = 422;
-               	response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
-            }
-        } else {
-        	response.code = 404
-           	response.body = '<object class="Error">{"message":"Command '+command.name+' not found","code":404}'
-        }
-        return response
-	}
-}
-
-expose(command)

package/src/worker-query.js

@@ -1,148 +0,0 @@
-import {expose} from "threads/worker"
-import JSONTag from "@muze-nl/jsontag"
-import pointer from 'json-pointer'
-import {_,from,not,anyOf,allOf} from 'array-where-select'
-import {deepFreeze} from './util.mjs'
-import {VM} from 'vm2'
-
-/**
- * Query Worker for threads.js library
- * returns JSONTag strings, since otherwise JSON.stringify is used
- * and type+attribute data gets lost
- * perhaps return [result,err] instead, so errors can be detected?
- */
-
-let dataspace, meta = {};
-
-function getDataSpace(path, dataspace) {
-    if (path.substring(path.length-1)==='/') {
-        //jsonpointer doesn't allow a trailing '/'
-        path = path.substring(0, path.length-1)
-    }
-    let result
-    if (path) {
-        //jsonpointer doesn't allow an empty pointer
-        try {
-            if (pointer.has(dataspace, path)) {
-                result = pointer.get(dataspace, path)
-            } else {
-                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
-            }
-        } catch(err) {
-            result = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
-        }
-    } else {
-        result = dataspace
-    }
-    return [result,path]
-}
-
-function linkReplacer(data, baseURL) {
-    let type = JSONTag.getType(data)
-    let attributes = JSONTag.getAttributes(data)
-    if (Array.isArray(data)) {
-        data = data.map((entry,index) => {
-            return linkReplacer(data[index], baseURL+index+'/')
-        })
-    } else if (type === 'link') {
-        // do nothing
-    } else if (data && typeof data === 'object') {
-        data = JSONTag.clone(data)
-        Object.keys(data).forEach(key => {
-            if (Array.isArray(data[key])) {
-                data[key] = new JSONTag.Link(baseURL+key+'/')
-            } else if (typeof data[key] === 'object') {
-                if (JSONTag.getType(data[key])!=='link') {
-                    let id=JSONTag.getAttribute(data[key], 'id')
-                    if (!id) {
-                        id = baseURL+key+'/'
-                    }
-                    data[key] = new JSONTag.Link(id)
-                }
-            }
-        })
-    }
-    return data
-}
-
-//@TODO: emit console events that server.mjs picks up
-function connectConsole(res) {
-    return {
-        log: function(...args) {
-//            res.append('X-Console-Log', joinArgs(args))
-        },
-        warning: function(...args) {
-//            res.append('X-Console-Warning', joinArgs(args))
-        },
-        error: function(...args) {
-//            res.append('X-Console-Error', joinArgs(args))            
-        }
-    }
-}
-
-const query = {
-	initialize(jsontag) {
-        if (!jsontag) { throw new Error('missing jsontag parameter')}
-		dataspace = JSONTag.parse(jsontag, null, meta)
-        deepFreeze(dataspace)
-        console.log('initialized query worker thread')
-        return true
-	},
-	runQuery(pointer, request, query=null) {
-        if (!pointer) { throw new Error('missing pointer parameter')}
-        if (!request) { throw new Error('missing request parameter')}
-		console.log('query',pointer)
-    	let response = {
-            jsontag: request.jsontag
-        }
-        let [result,path] = getDataSpace(pointer, dataspace)
-
-        if (query) {
-            // @todo add text search: https://github.com/nextapps-de/flexsearch
-            // @todo add tree walk map/reduce/find/filter style functions
-            // @todo add arc tree dive function?
-            // @todo replace VM with V8 isolate
-            const vm = new VM({
-                timeout: 1000,
-                allowAsync: false,
-                sandbox: {
-                    root: dataspace, //@TODO: if we don't pass the root, we can later shard
-                    data: result,
-                    meta: meta,
-                    _: _,
-                    from: from,
-                    not: not,
-                    anyOf: anyOf,
-                    allOf: allOf,
-//                    console: connectConsole(res),
-                    JSONTag: JSONTag,
-                    request: request
-                },
-                wasm: false
-            })
-            try {
-                result = vm.run(query)
-                let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
-                console.log(`(${used} MB)`);
-            } catch(err) {
-                console.log(err)
-                response.code = 422;
-                if (request.jsontag) {
-                	response.body = '<object class="Error">{"message":'+originalJSON.stringify(''+err)+',"code":422}'
-                } else {
-                	response.body = JSON.stringify({message:err, code: 422})
-                }
-            }
-        }
-        if (!response.code) {
-	        if (response.jsontag) {
-	        	response.body = JSONTag.stringify(linkReplacer(result, path+'/'))
-	        } else {
-	        	response.body = JSON.stringify(result)
-	        }
-	    }
-        return response
-	}
-}
-
-expose(query)