@muze-nl/simplystore 0.9.3 → 0.10.2

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.9.3",
+	"version": "0.10.2",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {
@@ -16,14 +16,13 @@
 	"bugs": "https://github.com/simplyedit/simplystore/issues",
 	"homepage": "https://github.com/simplyedit/simplystore#readme",
 	"dependencies": {
-		"@muze-nl/jaqt": "^0.10.3",
+		"@muze-nl/jaqt": "^0.10.6",
 		"@muze-nl/jsontag": "^0.10.4",
-		"@muze-nl/od-jsontag": "^0.4.5",
+		"@muze-nl/od-jsontag": "^0.4.6",
 		"codemirror": "^6.0.1",
-		"express": "^4.18.1",
-		"json-pointer": "^0.6.2",
-		"vm2": "^3.9.13",
-		"write-file-atomic": "^5.0.1"
+		"express": "^5.2.1",
+		"vm2": "^3.10.5",
+		"write-file-atomic": "^7.0.1"
 	},
 	"devDependencies": {
 		"@eslint/js": "^9.36.0",

package/scripts/convert.mjs

@@ -1,15 +1,42 @@
 import JSONTag from '@muze-nl/jsontag'
 import serialize, { stringify } from '@muze-nl/od-jsontag/src/serialize.mjs'
+import Parser from '@muze-nl/od-jsontag'
 import fs from 'node:fs'
+import path from 'node:path'
+
+const __dirname = import.meta.dirname;
 
 if (process.argv.length<=3) {
-	console.log('usage: node ./convert.mjs {inputfile} {outputfile}')
+	console.log('usage: node ./convert.mjs {inputfile} {outputfile} {indexlib?}')
 	process.exit()
 }
 
 // parse command line
 let inputFile = process.argv[2]
 let outputFile = process.argv[3]
+let indexFile = process.argv[4]
+if (indexFile && indexFile[0]!='/') {
+	indexFile = process.cwd()+'/'+indexFile
+} else if (!indexFile) {
+	indexFile = __dirname+'/../src/index.mjs'
+}
+let schemaFile = process.argv[5]
+if (schemaFile && schemaFile[0]!='/') {
+	schemaFile = process.cwd()+'/'+schemaFile
+}
+
+const index = await import(indexFile).then(mod => {
+    return mod.default
+})
+
+// now create indexes
+console.log('Using index library:', indexFile)
+
+
+let schema = {}
+if (schemaFile) {
+	schema = JSONTag.parse(fs.readFileSync(schemaFile, 'utf-8'))
+}
 
 // load file
 let input = fs.readFileSync(inputFile, 'utf-8')
@@ -17,9 +44,43 @@ let input = fs.readFileSync(inputFile, 'utf-8')
 // parse jsontag
 let data = JSONTag.parse(input)
 
+console.log('input data parsed')
 // write resultset to output
 let strData = stringify(serialize(data))
 
-fs.writeFileSync(outputFile, strData)
+console.log('od-jsontag created')
 
+// indexes need the position data which is only available after
+// parsing the od-jsontag data
+const parser = new Parser('https://localhost/',false) // allow mutations
+const odData = parser.parse(strData)
+
+console.log('offsets parsed')
+
+let meta = {
+	index: {
+		id: new Map()
+	},
+	schema,
+	resultArray: parser.meta.resultArray,
+	data: path.dirname(outputFile)
+}
+
+for (let ob of meta.resultArray) {
+	let id = JSONTag.getAttribute(ob, 'id')
+	if (id) {
+		meta.index.id.set(id, ob)
+	}
+}
+
+console.log('Indexing...')
+//FIXME: offset index can only be calculated by parsing serialized odData
+//but index.create updates that data, so previous information is no longer correct
+//TODO: split index creation in internal (updates odData) and external (creates index files)
+index.create(odData, meta)
+console.log('Indexes created')
+
+strData = stringify(serialize(odData))
+
+fs.writeFileSync(outputFile, strData)
 console.log('Converted data written to ',outputFile)

package/scripts/index.id.json

@@ -0,0 +1 @@
+{}

package/scripts/index.offset.json

@@ -0,0 +1 @@
+{}

package/src/command-worker-module.mjs

@@ -1,10 +1,11 @@
 import JSONTag from '@muze-nl/jsontag'
-import {getIndex, resultSet} from '@muze-nl/od-jsontag/src/symbols.mjs'
+import {getIndex, resultSet, isChanged} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 import writeFileAtomic from 'write-file-atomic'
 
 let commands = {}
+let index = {}
 let resultArr = []
 let dataspace
 let datafile, basefile, extension
@@ -79,6 +80,9 @@ export async function initialize(task) {
     commands = await import(task.commandsFile).then(mod => {
         return mod.default
     })
+    index = await import(task.indexFile).then(mod => {
+        return mod.default
+    })
 }
 
 export default async function runCommand(commandStr, request) {
@@ -94,6 +98,12 @@ export default async function runCommand(commandStr, request) {
             commands[task.name](dataspace, task, request, metaProxy)
             //TODO: if command/task makes no changes, skip updating data.jsontag and writing it, skip response.data
         
+            const changes = meta.resultArray.filter(e => e[isChanged])
+            //FIXME: new entities should also report isChanged = true
+            if (changes.length) {
+                changes.uuid = task.id
+                index.update(dataspace, meta, changes)
+            }
             const uint8sab = serialize(dataspace, {meta, changes: true}) // serialize only changes
             response.data = uint8sab
             response.meta = {

package/src/index.id.mjs

@@ -0,0 +1,45 @@
+import fs from 'fs'
+import JSONTag from '@muze-nl/jsontag'
+import { getIndex } from '@muze-nl/od-jsontag/src/symbols.mjs'
+
+export default {
+	create(data, meta) {
+		console.log('creating '+meta.data+'/index.id.json')
+		// jsontag parse automatically fills meta.index.id, so no need to create anything
+		// just store meta.index.id in index.id.json
+		const index = {}
+		for (const key of meta.index.id.keys()) {
+			if (!key) {
+				continue
+			}
+			const entity = meta.index.id.get(key)
+			if (!entity) {
+				continue
+			}
+			index[key] = entity[getIndex]
+		}
+		fs.writeFileSync(meta.data+'/index.id.json', JSON.stringify(index))
+	},
+	update(data, meta, changes) {
+		if (!changes.length) {
+			return
+		}
+		const index = {}
+		for (const entry of changes) {
+			const id = JSONTag.getAttribute(entry, 'id')
+			if (id) {
+				index[id] = entry[getIndex]
+			}
+		}
+		fs.writeFileSync(meta.data+'/index.id.'+changes.uuid+'.json', JSON.stringify(index))
+	},
+	load(uuid=null) {
+		let filename
+		if (!uuid) {
+			filename = 'index.id.json'
+		} else {
+			filename = 'index.id.'+filename+'.json'
+		}
+		return JSON.parse(fs.readFileSync(meta.data+filename))
+	}
+}

package/src/index.mjs

@@ -0,0 +1,19 @@
+import idIndex from './index.id.mjs'
+import offsetIndex from './index.offset.mjs'
+
+export default {
+	create(data, meta) {
+		idIndex.create(data, meta)
+		offsetIndex.create(data, meta)
+	},
+	update(data, meta, changes) {
+		idIndex.update(data, meta, changes)
+		offsetIndex.update(data, meta, changes)
+	},
+	load(meta, uuid=null) {
+		return {
+			id: idIndex.load(uuid),
+		    offset: offsetIndex.load(uuid)
+		}
+	}
+}

package/src/index.offset.mjs

@@ -0,0 +1,39 @@
+import fs from 'fs'
+import JSONTag from '@muze-nl/jsontag'
+import { getIndex, position } from '@muze-nl/od-jsontag/src/symbols.mjs'
+
+export default {
+	create(data, meta) {
+		console.log('creating '+meta.data+'/index.offset.json')
+		// jsontag parse automatically fills meta.index.offset, so no need to create anything
+		const index = {}
+		const max = meta.resultArray.length
+		for (let i=0;i<max;i++) {
+			const entity = meta.resultArray[i]
+			index[i] = [ entity[position].start, entity[position].end ]
+		}
+		fs.writeFileSync(meta.data+'/index.offset.json', JSON.stringify(index))
+	},
+	update(data, meta, changes) {
+		if (!changes.length) {
+			return
+		}
+		const index = {}
+		for (const entry of changes) {
+			let pos = entry[position]
+			if (pos) {
+				index[entry[getIndex]] = [ pos.start, pos.end ]
+			}
+		}
+		fs.writeFileSync(meta.data+'/index.offset.'+changes.uuid+'.json', JSON.stringify(index))
+	},
+	load(uuid=null) {
+		let filename
+		if (!uuid) {
+			filename = 'index.offset.json'
+		} else {
+			filename = 'index.offset.'+filename+'.json'
+		}
+		return JSON.parse(fs.readFileSync(meta.data+filename))
+	}
+}

package/src/load-worker.mjs

@@ -2,25 +2,39 @@ import { parentPort } from 'node:worker_threads'
 import JSONTag from '@muze-nl/jsontag'
 import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
 import fs from 'fs'
+import path from 'path'
 import serialize from '@muze-nl/od-jsontag/src/serialize.mjs'
 
 const parser = new Parser()
-parentPort.on('message', (files) => {
+let index = {}
+
+parentPort.on('message', async (files) => {
 	let meta = {
 		index: {
 			id: new Map()
 		}
 	}
 
+    index = await import(files.indexFile).then(mod => {
+        return mod.default
+    })
 	const extension = files.dataFile.split('.').pop()
 	const basefile = files.dataFile.substring(0, files.dataFile.length - (extension.length + 1)) //+1 for . character
-
+	meta.data = path.dirname(basefile)
 	let count = 0
 	let data
 	let jsontag
 	let datafile = files.dataFile
 	let commands = files.commands
 	commands.push('done')
+	// TODO
+	// - only load index files
+	// - for each command id
+	// - load files as raw bytes
+	// - index.id.*.jsontag and index.offset.*.jsontag to create proxies that will get the correct offset on access
+	// - do the same for resultSet[0] - the dataspace root entity
+	// don't parse entire files with od-jsontag
+	// add version info in proxies with a symbol to get that information
 	do {
 		if (fs.existsSync(datafile)) {
 			jsontag = fs.readFileSync(datafile)

package/src/query-worker-module.mjs

@@ -1,10 +1,9 @@
-import pointer from 'json-pointer'
 import {VM} from 'vm2'
 import { memoryUsage } from 'node:process'
 import JSONTag from '@muze-nl/jsontag'
 import {source} from '@muze-nl/od-jsontag/src/symbols.mjs'
 import Parser from '@muze-nl/od-jsontag/src/parse.mjs'
-import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,distinct} from '@muze-nl/jaqt'
+import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,first,distinct} from '@muze-nl/jaqt'
 import process from 'node:process'
 
 let dataspace
@@ -56,7 +55,7 @@ const tasks = {
         return true
     },
     query: async (task) => {
-        return runQuery(task.req.path, task.req, task.req.body)
+        return runQuery(task.req.path, task.req, task.req.body, task.timeout)
     },
     memoryUsage: async () => {
         let result = memoryUsage()
@@ -67,7 +66,7 @@ const tasks = {
 
 export default tasks
 
-export function runQuery(pointer, request, query) {
+export function runQuery(pointer, request, query, timeout=1000) {
     if (!pointer) { throw new Error('missing pointer parameter')}
     if (!request) { throw new Error('missing request parameter')}
     let response = {
@@ -79,7 +78,7 @@ export function runQuery(pointer, request, query) {
         // @todo add text search: https://github.com/nextapps-de/flexsearch
         // @todo replace VM with V8 isolate
         const vm = new VM({
-            timeout: 1000,
+            timeout: timeout,
             allowAsync: false,
             sandbox: {
                 root: dataspace, //@TODO: if we don't pass the root, we can later shard
@@ -99,6 +98,7 @@ export function runQuery(pointer, request, query) {
                 min,
                 many,
                 one,
+                first,
                 distinct,
 //                    console: connectConsole(res),
                 JSONTag,
@@ -171,24 +171,18 @@ function parseAllObjects(o, reset=true) {
 }
 
 export function getDataSpace(path, dataspace) {
-    if (path.substring(path.length-1)==='/') {
-        //jsonpointer doesn't allow a trailing '/'
+    if (path.substring(path.length-1)=='/') {
         path = path.substring(0, path.length-1)
-    }
-    let result
-    if (path) {
-        //jsonpointer doesn't allow an empty pointer
-        try {
-            if (pointer.has(dataspace, path)) {
-                result = pointer.get(dataspace, path)
-            } else {
-                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
-            }
-        } catch(err) {
-            result = JSONTag.parse('<object class="Error">{"message":'+JSON.stringify(err.message)+', "code":500}')
+    } 
+    const pointer = path.split('/')
+    let result = dataspace
+    for (const part of pointer) {
+        if (part && result) {
+            result = result[part]
         }
-    } else {
-        result = dataspace
+    }
+    if (result===undefined) {
+        result = JSONTag.parse(`<object class="Error">{"message":"Path Not found", "code":404, "path":"${path}"}`)
     }
     return [result,path]
 }

package/src/server.mjs

@@ -27,11 +27,14 @@ async function main(options) {
     const loadWorker    = options.loadWorker    || __dirname+'/src/load-worker.mjs'
     const commandWorker = options.commandWorker || __dirname+'/src/command-worker.mjs'
     const commandsFile  = options.commandsFile  || __dirname+'/src/commands.mjs'
+    const indexFile     = options.indexFile     || __dirname+'/src/index.mjs'
     const commandLog    = options.commandLog    || './command-log.jsontag'
     const commandStatus = options.commandStatus || './command-status.jsontag'
     const access        = options.access        || null
+    const timeout       = options.timeout       || 1000
+    const slowTimeout   = options.slowTimeout   || 10000
 
-    server.use(express.static(wwwroot))
+    server.get('/', serveHomepage)
 
     // allow access to raw body, used to parse a query send as post body
     server.use(express.raw({
@@ -59,17 +62,33 @@ async function main(options) {
                 body: jsontagBuffers,
                 meta,
                 access
-            }
+            },
+            timeout
         }
     }
 
+    const slowQueryWorkerInitTask = () => {
+        let result = queryWorkerInitTask()
+        result.timeout = slowTimeout
+        return result
+    }
+
     let queryWorkerPool = new WorkerPool(maxWorkers, queryWorker, queryWorkerInitTask())
+    let slowQueryWorkerPool = new WorkerPool(1, queryWorker, slowQueryWorkerInitTask())
     let commandWorkerInstance
 
-    server.get('/query/*', handleGetQuery)
-    server.post('/query/*', handlePostQuery)
-    server.post('/command', handlePostCommand)
-    server.get('/command/:id', handleGetCommand)
+    server.get('/query/', (req,next) => handleGetQuery(req,next))
+    server.get('/query/*remainder', (req,next) => handleGetQuery(req,next))
+    server.get('/slowquery/', (req,next) => handleSlowGetQuery(req,next))
+    server.get('/slowquery/*remainder', (req,next) => handleSlowGetQuery(req,next))
+    server.post('/query/', (req,next) => handlePostQuery(req,next))
+    server.post('/query/*remainder', (req,next) => handlePostQuery(req,next))
+    server.post('/slowquery/', (req,next) => handleSlowPostQuery(req,next))
+    server.post('/slowquery/*remainder', (req,next) => handleSlowPostQuery(req,next))
+    server.post('/command', (req,next) => handlePostCommand(req,next))
+    server.get('/command/:id', (req,next) => handleGetCommand(req,next))
+
+    server.use(express.static(wwwroot))
 
     try {
         await fetch(`http://localhost:${port}`, {
@@ -95,6 +114,11 @@ async function main(options) {
 
     /* ------ */
 
+    function serveHomepage(req, res) {
+        res.setHeader('content-type', 'text/html');
+        res.send(fs.readFileSync(wwwroot+'/home.html'))
+    }
+
     function loadCommandStatus(commandStatusFile) {
         let status = new Map()
         if (fs.existsSync(commandStatusFile)) {
@@ -134,6 +158,7 @@ async function main(options) {
                             meta,
                             data:jsontagBuffers,
                             commandsFile,
+                            indexFile,
                             datafile
                         })
                         break;
@@ -158,12 +183,15 @@ async function main(options) {
                 reject(error)
                 worker.terminate()
             })
-            worker.postMessage({dataFile:datafile,schemaFile,commands})
+            worker.postMessage({dataFile:datafile,indexFile,schemaFile,commands})
         })
     }
 
-    async function handleGetQuery(req, res) {
+    async function handleGetQuery(req, res, pool=null) {
         let start = Date.now()
+        if (!pool) {
+            pool = queryWorkerPool
+        }
         if ( !accept(req,res,
             ['application/jsontag','application/json','text/html','text/javascript','image/*'], 
             function(req, res, accept) {
@@ -182,7 +210,7 @@ async function main(options) {
             // done
             return
         }
-        let path = req.path.substr(6) // cut '/query'
+        let path = req.params.remainder?.join('/') || '/'
         console.log('query',path)
         let request = {
             method: req.method,
@@ -194,7 +222,7 @@ async function main(options) {
             request.jsontag = true
         }
         try {
-            let result = await queryWorkerPool.run('query', request)
+            let result = await pool.run('query', request, { timeout })
             sendResponse(result, res)
         } catch(error) {
             sendError(error, res)
@@ -203,7 +231,17 @@ async function main(options) {
         console.log(path, (end-start), process.memoryUsage())
     }
 
-    async function handlePostQuery(req,res) {
+    async function handleSlowGetQuery(req, res) {
+        return handleGetQuery(req, res, slowQueryWorkerPool)
+    }
+
+    async function handlePostQuery(req,res,pool=null, slowTimeout=null) {
+        if (!pool) {
+            pool = queryWorkerPool
+        }
+        if (!slowTimeout) {
+            slowTimeout = timeout
+        }
         let start = Date.now()
         if ( !accept(req,res,
             ['application/jsontag','application/json']) 
@@ -211,7 +249,7 @@ async function main(options) {
             sendError({code:406, message:'Not Acceptable',accept:['application/json','application/jsontag']},res)
             return
         }
-        let path = req.path.substr(6) // cut '/query'
+        let path = req.params.remainder?.join('/') || '/'
         let request = {
             method: req.method,
             url: req.originalUrl,
@@ -223,7 +261,7 @@ async function main(options) {
             request.jsontag = true
         }
         try {
-            let result = await queryWorkerPool.run('query', request)
+            let result = await pool.run('query', request, {slowTimeout})
             sendResponse(result, res)
         } catch(error) {
             sendError(error, res)
@@ -233,6 +271,10 @@ async function main(options) {
 //        queryWorkerPool.memoryUsage()
     }
 
+    async function handleSlowPostQuery(req, res) {
+        return handlePostQuery(req, res, slowQueryWorkerPool, slowTimeout)
+    }
+
     async function handlePostCommand(req, res) {
         let commandId = checkCommand(req, res)
         if (!commandId) {
@@ -253,6 +295,7 @@ async function main(options) {
                 meta,
                 data:jsontagBuffers,
                 commandsFile,
+                indexFile,
                 datafile                
             })
             let result
@@ -326,13 +369,15 @@ async function main(options) {
                             if (data.data) { // data has changed, commands may do other things instead of changing data
                                 jsontagBuffers.push(data.data) // push changeset to jsontagBuffers so that new query workers get all changes from scratch
                                 Object.assign(meta, data.meta)
-                                queryWorkerPool.update({
+                                const updateTask = {
                                     name: 'update',
                                     req: {
                                         body: jsontagBuffers[jsontagBuffers.length-1], // only add the last change, update tasks for earlier changes have already been sent
                                         meta
                                     }
-                                })
+                                }
+                                queryWorkerPool.update(updateTask)
+                                slowQueryWorkerPool.update(updateTask)
                             }
                             appendFile(commandStatus, JSONTag.stringify(Object.assign({command:command.id}, s)))
                             mainResolve(s)

package/src/workerPool.mjs

@@ -68,9 +68,9 @@ export default class WorkerPool extends EventEmitter {
     worker.postMessage(this.initTask);
   }
 
-  async run(name, req) {
+  async run(name, req, options={}) {
     return new Promise((resolve, reject) => {
-      this.runTask({name,req}, (error, result) => {
+      this.runTask({name,req,...options}, (error, result) => {
         if (error) {
           return reject(error)
         }

package/www/home.html

@@ -0,0 +1,21 @@
+<!doctype html>
+<meta charset="utf-8">
+<link rel="stylesheet" href="/assets/css/style.css">
+<title>SimplyStore</title>
+<body>
+	<h1>SimplyStore</h1>
+	<h2>In memory database and API</h2>
+	<p>This is a placeholder page. To run queries go to <a href="query/">the <code>/query</code> endpoint</a></p>
+	<p>You can override this placeholder page by adding this to your SimplyStore server code:</p>
+	<pre> 
+	import SimplyStore from '@muze-nl/simplystore'
+
+	SimplyStore.get('/', function(req, res) {
+	        res.setHeader('Content-Type', 'text/html')
+	        res.send('Your homepage content')
+	})
+
+	SimplyStore.run()
+	</pre>
+</body>
+