@muze-nl/simplystore 0.6.11 → 0.6.12

package/README.md

@@ -152,18 +152,14 @@ In addition, SimplyStore is meant to be a real-world testcase for JSONTag.
 <a name="roadmap"></a>
 ## Roadmap
 
-- allow changes to dataset by creating a new root
-- command handling with crud commands and command log
-- backup current dataset to JSONTag file
-- on startup check if any commands in the log haven't been resolved, if so run them
-
-- improved web client with type-specific views and form elements
-
-- add support for metadata on each JSON pointer path (or better: each object)
-- allow custom templates, instead of the default index.html
-- add support for access control, based on webid / openid connect
-- switch from VM2 to V8-isolate, which is more secure
-- switch the server runtime to Rust, so SimplyStore can share immutable data between threads
+- [x] allow changes to dataset by creating a new root
+- [x] command handling with crud commands and command log
+- [x] backup current dataset to JSONTag file
+- [x] on startup check if any commands in the log haven't been resolved, if so run them
+- [ ] improved web client with type-specific views and form elements
+- [ ] allow custom templates, instead of the default index.html
+- [ ] add support for access control, based on webid / openid connect
+- [ ] switch from VM2 to V8-isolate or QuickJS, which is more secure
 
 <a name="license"></a>
 ## License

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.6.11",
+	"version": "0.6.12",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -17,7 +17,7 @@
 	"homepage": "https://github.com/simplyedit/simplystore#readme",
 	"dependencies": {
 		"@muze-nl/jsontag": "^0.9.6",
-		"@muze-nl/od-jsontag": "^0.1.7",
+		"@muze-nl/od-jsontag": "^0.2.6",
 		"codemirror": "^6.0.1",
 		"express": "^4.18.1",
 		"jaqt": "^0.6.2",

package/src/command-worker-module.mjs

@@ -1,10 +1,8 @@
 import JSONTag from '@muze-nl/jsontag'
-import {source, isChanged, getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
+import {getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
-import serialize, {stringify} from '@muze-nl/od-jsontag/src/serialize.mjs'
-import * as FastJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
+import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 import writeFileAtomic from 'write-file-atomic'
-import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min} from 'jaqt'
 
 let commands = {}
 let resultArr = []
@@ -54,7 +52,9 @@ export const metaIdProxy = {
 }
 
 export async function initialize(task) {
-    dataspace = parse(task.data, task.meta, false) // false means mutable
+    for(let jsontag of task.data) {
+        dataspace = parse(jsontag, task.meta, false) // false means mutable
+    }
     resultArr = dataspace[resultSet]
     meta = task.meta
     metaProxy.index.id = metaIdProxy
@@ -76,9 +76,9 @@ export default async function runCommand(commandStr, request) {
         let time = Date.now()
         commands[task.name](dataspace, task, request, metaProxy)
         //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
-        FastJSONTag.setAttribute(dataspace, 'command', task.id)
+        JSONTag.setAttribute(dataspace, 'command', task.id)
 
-        const uint8sab = serialize(dataspace)
+        const uint8sab = serialize(dataspace, {meta, changes: true}) // serialize only changes
         response.data = uint8sab
         response.meta = {
             index: {
@@ -86,7 +86,11 @@ export default async function runCommand(commandStr, request) {
             }
         }
         //TODO: write data every x commands or x minutes, in seperate thread
-        await writeFileAtomic(datafile, uint8sab)
+
+        let newfilename = datafile + (meta.parts ? '.'+meta.parts : '')
+        await writeFileAtomic(newfilename, uint8sab)
+        meta.parts++
+        response.meta.parts = meta.parts
         let end = Date.now()
         console.log('task time',end-time)
     } else {

package/src/load-worker.mjs

@@ -1,33 +1,34 @@
 import { parentPort } from 'node:worker_threads'
-import JSONTag from '@muze-nl/jsontag'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
 import fs from 'fs'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
-import {source, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 
 parentPort.on('message', datafile => {
-	const jsontag = fs.readFileSync(datafile)
 	let meta = {
 		index: {
 			id: new Map()
 		}
 	}
-	const data = parse(jsontag)
-	const resultArr = data[resultSet]
 
-	// od-jsontag/parse doesn't create meta.index.id, so do that here
-	let length = resultArr.length
-	for (let i=0; i<length; i++) {
-		let id=JSONTag.getAttribute(resultArr[i][source],'id')
-		if (id) {
-			meta.index.id.set(id,i)
-		}
-	}
+	let count = 0
+	let basefile = datafile
+	let data 
+	let jsontag
+	let tempMeta = {}
 
-	const sab = serialize(data)
+	do {
+		jsontag = fs.readFileSync(datafile)
+		data = parse(jsontag, tempMeta) // tempMeta is needed to combine the resultArray, using meta conflicts with meta.index.id
+		count++
+		datafile = basefile + '.' + count
+	} while(fs.existsSync(datafile))
+	meta.parts = count
+
+	const sab = serialize(data, {meta})
 
 	parentPort.postMessage({
 		data: sab,
 		meta
 	})
+
 })

package/src/query-worker-module.mjs

@@ -4,7 +4,6 @@ import { memoryUsage } from 'node:process'
 import JSONTag from '@muze-nl/jsontag'
 import {source, isProxy, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import parse from '@muze-nl/od-jsontag/src/parse.mjs'
-import * as FastJSONTag from '@muze-nl/od-jsontag/src/jsontag.mjs'
 import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min} from 'jaqt'
 
 let resultArr = []
@@ -32,14 +31,27 @@ const tasks = {
         if (task.req.access) {
             task.req.access = await import(task.req.access)
             task.req.access = task.req.access.default
+            meta.access = task.req.access
         }
-        dataspace = parse(task.req.body, { access: task.req.access })
-        resultArr = dataspace[resultSet]
-        meta = task.req.meta
+        if (task.req.meta.index) {
+            meta.index = task.req.meta.index
+        }
+        for (let sab of task.req.body) { //body contains an array of sharedArrayBuffers with initial data and changes
+            dataspace = parse(sab, meta)
+        }
+        resultArr = meta.resultArray
         metaProxy.index.id = metaIdProxy
         //@TODO: add meta.index.references? and baseURL
         return true
     },
+    update: async (task) => {
+        if (task.req.meta.index) {
+            meta.index = task.req.meta.index
+        }
+        dataspace = parse(task.req.body, meta) //update only has a single changeset
+        resultArr = meta.resultArray
+        return true
+    },
     query: async (task) => {
         return runQuery(task.req.path, task.req, task.req.body)
     },
@@ -83,7 +95,7 @@ export function runQuery(pointer, request, query) {
                 max,
                 min,    
 //                    console: connectConsole(res),
-                JSONTag: FastJSONTag,
+                JSONTag,
                 request
             },
             wasm: false
@@ -145,8 +157,8 @@ export function getDataSpace(path, dataspace) {
 }
 
 export function linkReplacer(data, baseURL) {
-    let type = FastJSONTag.getType(data)
-    let attributes = FastJSONTag.getAttributes(data)
+    let type = JSONTag.getType(data)
+    let attributes = JSONTag.getAttributes(data)
     if (Array.isArray(data)) {
         data = data.map((entry,index) => {
             return linkReplacer(data[index], baseURL+index+'/')
@@ -162,8 +174,8 @@ export function linkReplacer(data, baseURL) {
             if (Array.isArray(data[key])) {
                 data[key] = new JSONTag.Link(baseURL+key+'/')
             } else if (data[key] && typeof data[key] === 'object') {
-                if (FastJSONTag.getType(data[key])!=='link') {
-                    let id=FastJSONTag.getAttribute(data[key], 'id')
+                if (JSONTag.getType(data[key])!=='link') {
+                    let id=JSONTag.getAttribute(data[key], 'id')
                     if (!id) {
                         id = baseURL+key+'/'
                     }

package/src/server.mjs

@@ -11,7 +11,7 @@ import writeFileAtomic from 'write-file-atomic'
 
 const server = express()
 const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url)))
-let jsontagBuffer = null
+let jsontagBuffers = null
 let meta = {}
 
 async function main(options) {
@@ -34,7 +34,8 @@ async function main(options) {
 
     // allow access to raw body, used to parse a query send as post body
     server.use(express.raw({
-        type: (req) => true // parse body on all requests
+        type: (req) => true, // parse body on all requests
+        limit: '50MB'
     }))
 
     function loadData() {
@@ -53,7 +54,7 @@ async function main(options) {
     }
     try {
         let data = await loadData()
-        jsontagBuffer = data.data
+        jsontagBuffers = [data.data]
         meta = data.meta
     } catch(err) {
         console.error('ERROR: SimplyStore cannot load '+datafile, err)
@@ -64,13 +65,14 @@ async function main(options) {
         return {
             name: 'init',
             req: {
-                body: jsontagBuffer,
+                body: jsontagBuffers,
                 meta,
                 access
             }
         }
     }
 
+
     let queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
 
     server.get('/query/*', async (req, res, next) => 
@@ -227,18 +229,18 @@ async function main(options) {
                         s = {code: 200, status: "done"}
                         status.set(command.id, s)
                         if (data.data) { // data has changed, commands may do other things instead of changing data
-                            jsontagBuffer = data.data
+                            jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
                             meta = data.meta
-                            // restart query workers with new data
-                            let oldPool = queryWorkerPool
-                            queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
-                            setTimeout(() => {
-                                oldPool.close()
-                            }, 2000)
+                            queryWorkerPool.update({
+                                name: 'update',
+                                req: {
+                                    body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
+                                    meta
+                                }
+                            })
                         }
                     }
                     let l = Object.assign({command:command.id}, s)
-                    console.log('appending status ',l,s)
                     appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                 }, 
                 //reject()
@@ -276,7 +278,7 @@ async function main(options) {
                 command:commandStr,
                 request,
                 meta,
-                data:jsontagBuffer,
+                data:jsontagBuffers,
                 commandsFile,
                 datafile                
             })

package/src/workerPool.mjs

@@ -19,7 +19,6 @@ class WorkerPoolTaskInfo extends AsyncResource {
 }
 
 //@TODO: only create new workers when needed, not immediately
-//@TODO: allow initialization of newly created workers
 
 export default class WorkerPool extends EventEmitter {
   constructor(numThreads, workerFile, initTask) {
@@ -29,13 +28,14 @@ export default class WorkerPool extends EventEmitter {
     this.initTask    = initTask
     this.workers     = []
     this.freeWorkers = []
-
+    this.priority    = new Map()
     for (let i = 0; i < numThreads; i++)
       this.addNewWorker();
   }
 
   addNewWorker() {
     const worker = new Worker(path.resolve(this.workerFile));
+    this.priority[worker] = []
     worker.on('message', (result) => {
       // In case of success: Call the callback that was passed to `runTask`,
       // remove the `TaskInfo` associated with the Worker, and mark it as free
@@ -44,8 +44,13 @@ export default class WorkerPool extends EventEmitter {
         worker[kTaskInfo].done(null, result);
         worker[kTaskInfo] = null;
       }
-      this.freeWorkers.push(worker);
-      this.emit(kWorkerFreedEvent);
+      if (this.priority[worker].length) {
+        let task = this.priority[worker].shift()
+        worker.postMessage(task)
+      } else {
+        this.freeWorkers.push(worker);
+        this.emit(kWorkerFreedEvent);
+      }
     });
     worker.on('error', (err) => {
       // In case of an uncaught exception: Call the callback that was passed to
@@ -86,6 +91,16 @@ export default class WorkerPool extends EventEmitter {
     worker.postMessage(task);
   }
 
+  update(task) {
+    for (let worker of this.workers) {
+      if (worker[kTaskInfo]) { // worker is busy
+        this.priority[worker].push(task) // push task on priority list for this specific worker
+      } else { // worker is free
+        worker.postMessage(task) // run task immediately
+      }
+    }
+  }
+
   memoryUsage() {
     for (let worker of this.freeWorkers) {
       worker[kTaskInfo] = new WorkerPoolTaskInfo((result) => {})