@muze-nl/simplystore 0.6.11 → 0.6.14

package/README.md

@@ -152,18 +152,14 @@ In addition, SimplyStore is meant to be a real-world testcase for JSONTag.
 <a name="roadmap"></a>
 ## Roadmap
 
-- allow changes to dataset by creating a new root
-- command handling with crud commands and command log
-- backup current dataset to JSONTag file
-- on startup check if any commands in the log haven't been resolved, if so run them
-
-- improved web client with type-specific views and form elements
-
-- add support for metadata on each JSON pointer path (or better: each object)
-- allow custom templates, instead of the default index.html
-- add support for access control, based on webid / openid connect
-- switch from VM2 to V8-isolate, which is more secure
-- switch the server runtime to Rust, so SimplyStore can share immutable data between threads
+- [x] allow changes to dataset by creating a new root
+- [x] command handling with crud commands and command log
+- [x] backup current dataset to JSONTag file
+- [x] on startup check if any commands in the log haven't been resolved, if so run them
+- [ ] improved web client with type-specific views and form elements
+- [ ] allow custom templates, instead of the default index.html
+- [ ] add support for access control, based on webid / openid connect
+- [ ] switch from VM2 to V8-isolate or QuickJS, which is more secure
 
 <a name="license"></a>
 ## License

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.6.11",
+	"version": "0.6.14",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -17,7 +17,7 @@
 	"homepage": "https://github.com/simplyedit/simplystore#readme",
 	"dependencies": {
 		"@muze-nl/jsontag": "^0.9.6",
-		"@muze-nl/od-jsontag": "^0.1.7",
+		"@muze-nl/od-jsontag": "^0.2.6",
 		"codemirror": "^6.0.1",
 		"express": "^4.18.1",
 		"jaqt": "^0.6.2",

package/src/command-worker-module.mjs

@@ -1,10 +1,8 @@
 import JSONTag from '@muze-nl/jsontag'
-import {source, isChanged, getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
+import {getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
-import serialize, {stringify} from '@muze-nl/od-jsontag/src/serialize.mjs'
-import * as FastJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
+import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 import writeFileAtomic from 'write-file-atomic'
-import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min} from 'jaqt'
 
 let commands = {}
 let resultArr = []
@@ -53,48 +51,65 @@ export const metaIdProxy = {
     }
 }
 
+const metaReadProxy = {
+    foreach: metaProxy.forEach,
+    get: metaProxy.get,
+    has: metaProxy.has,
+    set: meta.set
+}
+
 export async function initialize(task) {
-    dataspace = parse(task.data, task.meta, false) // false means mutable
+    for(let jsontag of task.data) {
+        dataspace = parse(jsontag, task.meta, false) // false means mutable
+    }
     resultArr = dataspace[resultSet]
     meta = task.meta
     metaProxy.index.id = metaIdProxy
     datafile = task.datafile
     commands = await import(task.commandsFile).then(mod => {
-        console.log('commands loaded:',Object.keys(mod.default))
         return mod.default
     })
 }
 
 export default async function runCommand(commandStr, request) {
-    let task = JSONTag.parse(commandStr, null, metaProxy)
-    if (!task.id) { throw new Error('missing command id')}
-    if (!task.name) { throw new Error('missing command name parameter')}
     let response = {
         jsontag: true
     }
-    if (commands[task.name]) {
-        let time = Date.now()
-        commands[task.name](dataspace, task, request, metaProxy)
-        //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
-        FastJSONTag.setAttribute(dataspace, 'command', task.id)
+    try {
+        let task = JSONTag.parse(commandStr, null, metaReadProxy)
+        if (!task.id) { throw new Error('missing command id')}
+        if (!task.name) { throw new Error('missing command name parameter')}
+        if (commands[task.name]) {
+            let time = Date.now()
+            commands[task.name](dataspace, task, request, metaProxy)
+            //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
+            JSONTag.setAttribute(dataspace, 'command', task.id)
+        
+            const uint8sab = serialize(dataspace, {meta, changes: true}) // serialize only changes
+            response.data = uint8sab
+            response.meta = {
+                index: {
+                    id: meta.index.id
+                }
+            }
+            //TODO: write data every x commands or x minutes, in seperate thread
 
-        const uint8sab = serialize(dataspace)
-        response.data = uint8sab
-        response.meta = {
-            index: {
-                id: meta.index.id
+            let newfilename = datafile + (meta.parts ? '.'+meta.parts : '')
+            await writeFileAtomic(newfilename, uint8sab)
+            meta.parts++
+            response.meta.parts = meta.parts
+            let end = Date.now()
+            console.log('task time',end-time)
+        } else {
+            console.error('Command not found', task.name)
+            throw {
+                code: 404,
+                message: "Command "+task.name+" not found"
             }
         }
-        //TODO: write data every x commands or x minutes, in seperate thread
-        await writeFileAtomic(datafile, uint8sab)
-        let end = Date.now()
-        console.log('task time',end-time)
-    } else {
-        console.error('Command not found', task.name)
-        throw {
-            code: 404,
-            message: "Command "+task.name+" not found"
-        }
+    } catch(err) {
+        console.error('task error', err)
+        throw err
     }
     return response
 }

package/src/load-worker.mjs

@@ -1,33 +1,34 @@
 import { parentPort } from 'node:worker_threads'
-import JSONTag from '@muze-nl/jsontag'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
 import fs from 'fs'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
-import {source, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 
 parentPort.on('message', datafile => {
-	const jsontag = fs.readFileSync(datafile)
 	let meta = {
 		index: {
 			id: new Map()
 		}
 	}
-	const data = parse(jsontag)
-	const resultArr = data[resultSet]
 
-	// od-jsontag/parse doesn't create meta.index.id, so do that here
-	let length = resultArr.length
-	for (let i=0; i<length; i++) {
-		let id=JSONTag.getAttribute(resultArr[i][source],'id')
-		if (id) {
-			meta.index.id.set(id,i)
-		}
-	}
+	let count = 0
+	let basefile = datafile
+	let data 
+	let jsontag
+	let tempMeta = {}
 
-	const sab = serialize(data)
+	do {
+		jsontag = fs.readFileSync(datafile)
+		data = parse(jsontag, tempMeta) // tempMeta is needed to combine the resultArray, using meta conflicts with meta.index.id
+		count++
+		datafile = basefile + '.' + count
+	} while(fs.existsSync(datafile))
+	meta.parts = count
+
+	const sab = serialize(data, {meta})
 
 	parentPort.postMessage({
 		data: sab,
 		meta
 	})
+
 })

package/src/query-worker-module.mjs

@@ -2,12 +2,11 @@ import pointer from 'json-pointer'
 import {VM} from 'vm2'
 import { memoryUsage } from 'node:process'
 import JSONTag from '@muze-nl/jsontag'
+import * as odJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
 import {source, isProxy, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
-import * as FastJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
 import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min} from 'jaqt'
 
-let resultArr = []
 let dataspace
 let meta = {}
 let metaProxy = {
@@ -15,11 +14,49 @@ let metaProxy = {
     }
 }
 
+function protect(target) {
+    if (target[source]) {
+        throw new Error('Data is immutable')
+    }
+}
+
+const myJSONTag = {
+    getAttribute: odJSONTag.getAttribute,
+    getAttributes: odJSONTag.getAttributes,
+    getType: odJSONTag.getType,
+    getTypeString: odJSONTag.getTypeString,
+    setAttribute: (target, name, value) => {
+        protect(target)
+        return odJSONTag.setAttribute(target, name, value)
+    },
+    setType: (target, type) => {
+        protect(target)
+        return odJSONTag.setType(target, type)
+    },
+    setAttributes: (target, attributes) => {
+        protect(target)
+        return odJSONTag.setAttributes(target, attributes)
+    },
+    addAttribute: (target, name, value) => {
+        protect(target)
+        return odJSONTag.addAttribute(target, name, value)
+    },
+    removeAttribute: (target, name) => {
+        protect(target)
+        return odJSONTag.removeAttribute(target, name)
+    },
+    getAttributesString: odJSONTag.getAttributesString,
+    isNull: odJSONTag.isNull,
+    clone: JSONTag.clone,
+    Link: JSONTag.Link,
+    Null: JSONTag.Null
+}
+
 const metaIdProxy = {
     get: (id) => {
         let index = meta.index.id.get(id)
         if (index || index===0) {
-            return resultArr[index]
+            return meta.resultArr[index]
         }
     },
     has: (id) => {
@@ -32,14 +69,25 @@ const tasks = {
         if (task.req.access) {
             task.req.access = await import(task.req.access)
             task.req.access = task.req.access.default
+            meta.access = task.req.access
+        }
+        if (task.req.meta.index) {
+            meta.index = task.req.meta.index
+        }
+        for (let sab of task.req.body) { //body contains an array of sharedArrayBuffers with initial data and changes
+            dataspace = parse(sab, meta)
         }
-        dataspace = parse(task.req.body, { access: task.req.access })
-        resultArr = dataspace[resultSet]
-        meta = task.req.meta
         metaProxy.index.id = metaIdProxy
         //@TODO: add meta.index.references? and baseURL
         return true
     },
+    update: async (task) => {
+        if (task.req.meta.index) {
+            meta.index = task.req.meta.index
+        }
+        dataspace = parse(task.req.body, meta) //update only has a single changeset
+        return true
+    },
     query: async (task) => {
         return runQuery(task.req.path, task.req, task.req.body)
     },
@@ -83,7 +131,7 @@ export function runQuery(pointer, request, query) {
                 max,
                 min,    
 //                    console: connectConsole(res),
-                JSONTag: FastJSONTag,
+                JSONTag: myJSONTag,
                 request
             },
             wasm: false
@@ -145,8 +193,8 @@ export function getDataSpace(path, dataspace) {
 }
 
 export function linkReplacer(data, baseURL) {
-    let type = FastJSONTag.getType(data)
-    let attributes = FastJSONTag.getAttributes(data)
+    let type = JSONTag.getType(data)
+    let attributes = JSONTag.getAttributes(data)
     if (Array.isArray(data)) {
         data = data.map((entry,index) => {
             return linkReplacer(data[index], baseURL+index+'/')
@@ -162,8 +210,8 @@ export function linkReplacer(data, baseURL) {
             if (Array.isArray(data[key])) {
                 data[key] = new JSONTag.Link(baseURL+key+'/')
             } else if (data[key] && typeof data[key] === 'object') {
-                if (FastJSONTag.getType(data[key])!=='link') {
-                    let id=FastJSONTag.getAttribute(data[key], 'id')
+                if (JSONTag.getType(data[key])!=='link') {
+                    let id=JSONTag.getAttribute(data[key], 'id')
                     if (!id) {
                         id = baseURL+key+'/'
                     }

package/src/server.mjs

@@ -11,7 +11,7 @@ import writeFileAtomic from 'write-file-atomic'
 
 const server = express()
 const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url)))
-let jsontagBuffer = null
+let jsontagBuffers = null
 let meta = {}
 
 async function main(options) {
@@ -34,7 +34,8 @@ async function main(options) {
 
     // allow access to raw body, used to parse a query send as post body
     server.use(express.raw({
-        type: (req) => true // parse body on all requests
+        type: (req) => true, // parse body on all requests
+        limit: '50MB'
     }))
 
     function loadData() {
@@ -53,7 +54,7 @@ async function main(options) {
     }
     try {
         let data = await loadData()
-        jsontagBuffer = data.data
+        jsontagBuffers = [data.data]
         meta = data.meta
     } catch(err) {
         console.error('ERROR: SimplyStore cannot load '+datafile, err)
@@ -64,13 +65,14 @@ async function main(options) {
         return {
             name: 'init',
             req: {
-                body: jsontagBuffer,
+                body: jsontagBuffers,
                 meta,
                 access
             }
         }
     }
 
+
     let queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
 
     server.get('/query/*', async (req, res, next) => 
@@ -227,22 +229,23 @@ async function main(options) {
                         s = {code: 200, status: "done"}
                         status.set(command.id, s)
                         if (data.data) { // data has changed, commands may do other things instead of changing data
-                            jsontagBuffer = data.data
+                            jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
                             meta = data.meta
-                            // restart query workers with new data
-                            let oldPool = queryWorkerPool
-                            queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
-                            setTimeout(() => {
-                                oldPool.close()
-                            }, 2000)
+                            queryWorkerPool.update({
+                                name: 'update',
+                                req: {
+                                    body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
+                                    meta
+                                }
+                            })
                         }
                     }
                     let l = Object.assign({command:command.id}, s)
-                    console.log('appending status ',l,s)
                     appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                 }, 
                 //reject()
                 (error) => {
+                    console.error(error)
                     let s = {status: "failed", code: error.code, message: error.message, details: error.details}
                     status.set(command.id, s)
                     appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
@@ -276,7 +279,7 @@ async function main(options) {
                 command:commandStr,
                 request,
                 meta,
-                data:jsontagBuffer,
+                data:jsontagBuffers,
                 commandsFile,
                 datafile                
             })

package/src/workerPool.mjs

@@ -19,7 +19,6 @@ class WorkerPoolTaskInfo extends AsyncResource {
 }
 
 //@TODO: only create new workers when needed, not immediately
-//@TODO: allow initialization of newly created workers
 
 export default class WorkerPool extends EventEmitter {
   constructor(numThreads, workerFile, initTask) {
@@ -29,13 +28,14 @@ export default class WorkerPool extends EventEmitter {
     this.initTask    = initTask
     this.workers     = []
     this.freeWorkers = []
-
+    this.priority    = new Map()
     for (let i = 0; i < numThreads; i++)
       this.addNewWorker();
   }
 
   addNewWorker() {
     const worker = new Worker(path.resolve(this.workerFile));
+    this.priority[worker] = []
     worker.on('message', (result) => {
       // In case of success: Call the callback that was passed to `runTask`,
       // remove the `TaskInfo` associated with the Worker, and mark it as free
@@ -44,8 +44,13 @@ export default class WorkerPool extends EventEmitter {
         worker[kTaskInfo].done(null, result);
         worker[kTaskInfo] = null;
       }
-      this.freeWorkers.push(worker);
-      this.emit(kWorkerFreedEvent);
+      if (this.priority[worker].length) {
+        let task = this.priority[worker].shift()
+        worker.postMessage(task)
+      } else {
+        this.freeWorkers.push(worker);
+        this.emit(kWorkerFreedEvent);
+      }
     });
     worker.on('error', (err) => {
       // In case of an uncaught exception: Call the callback that was passed to
@@ -86,6 +91,16 @@ export default class WorkerPool extends EventEmitter {
     worker.postMessage(task);
   }
 
+  update(task) {
+    for (let worker of this.workers) {
+      if (worker[kTaskInfo]) { // worker is busy
+        this.priority[worker].push(task) // push task on priority list for this specific worker
+      } else { // worker is free
+        worker.postMessage(task) // run task immediately
+      }
+    }
+  }
+
   memoryUsage() {
     for (let worker of this.freeWorkers) {
       worker[kTaskInfo] = new WorkerPoolTaskInfo((result) => {})