@muze-nl/simplystore 0.4.7 → 0.5.1

package/src/fastStringify.mjs

@@ -0,0 +1,200 @@
+import JSONTag from '@muze-nl/jsontag';
+import {source,isProxy,getIndex, getBuffer} from './symbols.mjs'
+
+// faststringify function for a fast parseable arraybuffer output
+// 
+const encoder = new TextEncoder()
+const decoder = new TextDecoder()
+
+export default function stringify(value, meta, skipLength=false, index) {
+	let resultArray = []
+	if (!meta) {
+		meta = {}
+	}
+	if (!meta.index) {
+		meta.index = {}
+	}
+	if (!meta.index.id) {
+		meta.index.id = new Map()
+	}
+	let references = new WeakMap()
+
+	const innerStringify = (value) => {
+		let indent = ""
+		let gap = ""
+
+		if (typeof space === "number") {
+			indent += " ".repeat(space)
+		} else if (typeof space === "string") {
+			indent = space
+		}
+
+		const encodeProperties = (obj) => {
+			return Object.getOwnPropertyNames(obj).map(prop => {
+				let enumerable = obj.propertyIsEnumerable(prop) ? '' : '#'
+				return enumerable+'"'+prop+'":'+str(prop, obj)
+			}).join(',')
+		}
+
+		const encodeEntries = (arr) => {
+			let result = arr.map((value,index) => {
+				return str(index, arr)
+			}).join(",")
+			return result
+		}
+
+		const createId = (value) => {
+			if (typeof crypto === 'undefined') {
+				console.error('JSONTag: cannot generate uuid, crypto support is disabled.')
+				throw new Error('Cannot create links to resolve references, crypto support is disabled')
+			}
+			if (typeof crypto.randomUUID === 'function') {
+				var id = crypto.randomUUID()
+			} else {
+				var id = ([1e7]+-1e3+-4e3+-8e3+-1e11).replace(/[018]/g, c =>
+					(c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> c / 4).toString(16)
+				);
+			}
+			JSONTag.setAttribute(value, 'id', id)
+			return id
+		}
+
+		const encoder = new TextEncoder()
+		const decoder = new TextDecoder()
+
+		const str = (key, holder) => {
+			let value = holder[key]
+			let result, updateReference
+			// if value is a valueProxy, just copy the input slice
+			if (value && !JSONTag.isNull(value) && value[isProxy]) {
+				if (index===0) {
+					resultArray.push(decoder.decode(value[getBuffer](index)))
+				}
+				return decoder.decode(value[getBuffer](index))
+			}
+			if (typeof value === 'undefined' || value === null) {
+				return 'null'
+			}
+			if (JSONTag.getType(value) === 'object' && !Array.isArray(value)) {
+				let id = JSONTag.getAttribute(value, 'id')
+				if (!references.has(value)) {
+					let reference = resultArray.length
+					updateReference = reference
+					references.set(value, updateReference)
+					resultArray.push('')
+					if (id && !meta.index.id.has(id)) {
+						meta.index.id.set(id, updateReference)
+					}
+				} else {
+					return '~'+references.get(value)
+				}
+			}
+			if (Array.isArray(value)) {
+				result = JSONTag.getTypeString(value) + "["+encodeEntries(value)+"]"
+			} else if (value instanceof Object) {
+				let typeString = JSONTag.getTypeString(value)
+				let type = JSONTag.getType(value)
+				switch (type) {
+					case 'string':
+					case 'decimal':
+					case 'money':
+					case 'link':
+					case 'text':
+					case 'blob':
+					case 'color':
+					case 'email':
+					case 'hash':
+					case 'duration':
+					case 'phone':
+					case 'url':
+					case 'uuid':
+					case 'date':
+					case 'time':
+					case 'datetime':
+						if (JSONTag.isNull(value)) {
+							value = 'null'
+						} else {
+							value = JSON.stringify(''+value)
+						}
+						result = typeString + value
+					break
+					case 'int':
+					case 'uint':
+					case 'int8':
+					case 'uint8':
+					case 'int16':
+					case 'uint16':
+					case 'int32':
+					case 'uint32':
+					case 'int64':
+					case 'uint64':
+					case 'float':
+					case 'float32':
+					case 'float64':
+					case 'timestamp':
+					case 'number':
+					case 'boolean':
+						if (JSONTag.isNull(value)) {
+							value = 'null'
+						} else {
+							value = JSON.stringify(value)
+						}
+						result = typeString + value
+					break
+					case 'array': 
+						let entries = encodeEntries(value) // calculate children first so parent references can add id attribute
+						result = typeString + '[' + entries + '}'
+					break
+					case 'object': 
+						if (JSONTag.isNull(value)) {
+							result = typeString + "null"
+						} else {
+							let props = encodeProperties(value); // calculate children first so parent references can add id attribute
+							result = typeString + '{' + props + '}'
+						}
+					break
+					default:
+						throw new Error(JSONTag.getType(value)+' type not yet implemented')
+					break
+				}
+			} else {
+				result = JSON.stringify(value)
+			}
+			if (typeof updateReference != 'undefined') {
+				resultArray[updateReference] = result
+				if (index!==updateReference) {
+					result = '~'+updateReference
+				}
+			}
+			return result
+		}
+
+		return str("", {"": value})
+	}
+		
+	const encode = (s) => {
+		if (skipLength) {
+			return s
+		}
+		let length = new Blob([s]).size
+		return '('+length+')'+s
+	}
+
+	innerStringify(value)
+	return resultArray.map(encode).join("\n")
+}
+
+export function stringToSAB(strData) {
+	const buffer = encoder.encode(strData)
+	const sab = new SharedArrayBuffer(buffer.length)
+	let uint8sab = new Uint8Array(sab)
+	uint8sab.set(buffer,0)
+	return uint8sab
+}
+
+export function resultSetStringify(resultSet) {
+	return resultSet.map((e,i) => {
+		let buffer = e[getBuffer](i)
+		return '('+buffer.length+')'+decoder.decode(buffer)
+	}).join("\n")
+}

package/src/load-worker.mjs

@@ -0,0 +1,33 @@
+import { parentPort } from 'node:worker_threads'
+import JSONTag from '@muze-nl/jsontag'
+import fastParse from './fastParse.mjs'
+import fs from 'fs'
+import {resultSetStringify,stringToSAB} from './fastStringify.mjs'
+import {source} from '../src/symbols.mjs'
+
+parentPort.on('message', datafile => {
+	const jsontag = fs.readFileSync(datafile)
+	let meta = {
+		index: {
+			id: new Map()
+		}
+	}
+	const data = fastParse(jsontag)
+
+	// fastParse doesn't create meta.index.id, so do that here
+	let length = data.length
+	for (let i=0; i<length; i++) {
+		let id=JSONTag.getAttribute(data[i][source],'id')
+		if (id) {
+			meta.index.id.set(id,i)
+		}
+	}
+
+	const strData = resultSetStringify(data)
+	const sab = stringToSAB(strData)
+
+	parentPort.postMessage({
+		data: sab,
+		meta
+	})
+})

package/src/{worker-query.mjs → query-worker-module.mjs}

@@ -1,96 +1,63 @@
-import JSONTag from "@muze-nl/jsontag"
-import pointer from 'json-pointer'
+import JSONTag from '@muze-nl/jsontag'
+import fastParse from './fastParse.mjs'
+import {source, isProxy} from './symbols.mjs'
 import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min} from 'array-where-select'
-import {deepFreeze} from './util.mjs'
+import pointer from 'json-pointer'
 import {VM} from 'vm2'
+import { memoryUsage } from 'node:process'
 
-let dataspace, meta = {};
-
-export function setDataspace(d, m) {
-    dataspace = d
-    if(m) {
-        meta = m
+let resultSet = []
+let dataspace
+let meta = {}
+let metaProxy = {
+    index: {
     }
 }
 
-export function getDataSpace(path, dataspace) {
-    if (path.substring(path.length-1)==='/') {
-        //jsonpointer doesn't allow a trailing '/'
-        path = path.substring(0, path.length-1)
-    }
-    let result
-    if (path) {
-        //jsonpointer doesn't allow an empty pointer
-        try {
-            if (pointer.has(dataspace, path)) {
-                result = pointer.get(dataspace, path)
-            } else {
-                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
-            }
-        } catch(err) {
-            result = JSONTag.parse('<object class="Error">{"message":'+JSON.stringify(err.message)+', "code":500}')
+const metaIdProxy = {
+    get: (id) => {
+        let index = meta.index.id.get(id)
+        if (index || index===0) {
+            return resultSet[index]
         }
-    } else {
-        result = dataspace
+    },
+    has: (id) => {
+        return meta.index.id.has(id)
     }
-    return [result,path]
 }
 
-export function linkReplacer(data, baseURL) {
-    let type = JSONTag.getType(data)
-    let attributes = JSONTag.getAttributes(data)
-    if (Array.isArray(data)) {
-        data = data.map((entry,index) => {
-            return linkReplacer(data[index], baseURL+index+'/')
-        })
-    } else if (type === 'link') {
-        // do nothing
-    } else if (data && typeof data === 'object') {
-        data = JSONTag.clone(data)
-        Object.keys(data).forEach(key => {
-            if (Array.isArray(data[key])) {
-                data[key] = new JSONTag.Link(baseURL+key+'/')
-            } else if (data[key] && typeof data[key] === 'object') {
-                if (JSONTag.getType(data[key])!=='link') {
-                    let id=JSONTag.getAttribute(data[key], 'id')
-                    if (!id) {
-                        id = baseURL+key+'/'
-                    }
-                    data[key] = new JSONTag.Link(id)
-                }
-            }
-        })
-    }
-    return data
+const FastJSONTag = {
+    getType: (obj) => JSONTag.getType(obj[source]),
+    getAttribute: (obj, attr) => JSONTag.getAttribute(obj[source],attr),
+    getAttributes: (obj) => JSONTag.getAttributes(obj[source]),
+    getAttributeString: (obj) => JSONTag.getAttributesString(obj[source]),
+    getTypeString: (obj) => JSONTag.getTypeString(obj[source])
 }
 
-//@TODO: emit console events that server.mjs picks up
-function connectConsole(res) {
-    return {
-        log: function(...args) {
-//            res.append('X-Console-Log', joinArgs(args))
-        },
-        warning: function(...args) {
-//            res.append('X-Console-Warning', joinArgs(args))
-        },
-        error: function(...args) {
-//            res.append('X-Console-Error', joinArgs(args))            
-        }
+const tasks = {
+	init: async (task) => {
+		resultSet = fastParse(task.req.body)
+		dataspace = resultSet[0]
+        meta = task.req.meta
+        metaProxy.index.id = metaIdProxy
+        //@TODO: add references and baseURL
+		return true
+	},
+	query: async (task) => {
+		return runQuery(task.req.path, task.req, task.req.body)
+	},
+    memoryUsage: async () => {
+        let result = memoryUsage()
+        console.log('memory',result)
+        return result
     }
 }
 
-export async function initialize(jsontag, m) {
-    if (!jsontag) { throw new Error('missing jsontag parameter')}
-	dataspace = jsontag
-    meta = m
-    deepFreeze(dataspace)
-    return true
-}
+export default tasks
 
-export function runQuery({pointer, request, query}) {
+export function runQuery(pointer, request, query) {
     if (!pointer) { throw new Error('missing pointer parameter')}
     if (!request) { throw new Error('missing request parameter')}
-	console.log('query',pointer)
 	let response = {
         jsontag: request.jsontag
     }
@@ -98,8 +65,6 @@ export function runQuery({pointer, request, query}) {
 
     if (query) {
         // @todo add text search: https://github.com/nextapps-de/flexsearch
-        // @todo add tree walk map/reduce/find/filter style functions
-        // @todo add arc tree dive function?
         // @todo replace VM with V8 isolate
         const vm = new VM({
             timeout: 1000,
@@ -107,7 +72,7 @@ export function runQuery({pointer, request, query}) {
             sandbox: {
                 root: dataspace, //@TODO: if we don't pass the root, we can later shard
                 data: result,
-                meta,
+                meta: metaProxy,
                 _,
                 from,
                 not,
@@ -119,15 +84,15 @@ export function runQuery({pointer, request, query}) {
                 count,
                 avg,
                 max,
-                min,
+                min,    
 //                    console: connectConsole(res),
-                JSONTag,
+                JSONTag: FastJSONTag,
                 request
             },
             wasm: false
         })
         try {
-            result = vm.run(query)
+            result = deProxy(vm.run(query))
             let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
             console.log(`(${used} MB)`);
         } catch(err) {
@@ -144,10 +109,100 @@ export function runQuery({pointer, request, query}) {
     }
     if (!response.code) {
         if (response.jsontag) {
-        	response.body = JSONTag.stringify(result)
+            try {
+            	response.body = JSONTag.stringify(result)
+            } catch(err) {
+                console.log(err)
+                response.code = 500
+                response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":500}'
+            }
         } else {
+            //@FIXME: replace recursive links
         	response.body = JSON.stringify(result)
         }
     }
     return response
-}
+}
+
+export function getDataSpace(path, dataspace) {
+    if (path.substring(path.length-1)==='/') {
+        //jsonpointer doesn't allow a trailing '/'
+        path = path.substring(0, path.length-1)
+    }
+    let result
+    if (path) {
+        //jsonpointer doesn't allow an empty pointer
+        try {
+            if (pointer.has(dataspace, path)) {
+                result = pointer.get(dataspace, path)
+            } else {
+                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+            }
+        } catch(err) {
+            result = JSONTag.parse('<object class="Error">{"message":'+JSON.stringify(err.message)+', "code":500}')
+        }
+    } else {
+        result = dataspace
+    }
+    return [result,path]
+}
+
+export function linkReplacer(data, baseURL) {
+    let type = JSONTag.getType(data)
+    let attributes = JSONTag.getAttributes(data)
+    if (Array.isArray(data)) {
+        data = data.map((entry,index) => {
+            return linkReplacer(data[index], baseURL+index+'/')
+        })
+    } else if (type === 'link') {
+        // do nothing
+    } else if (data && typeof data === 'object') {
+        if (data[source]) {
+            data = data[source]
+        }
+        data = JSONTag.clone(data)
+        Object.keys(data).forEach(key => {
+            if (Array.isArray(data[key])) {
+                data[key] = new JSONTag.Link(baseURL+key+'/')
+            } else if (data[key] && typeof data[key] === 'object') {
+                if (JSONTag.getType(data[key])!=='link') {
+                    let id=JSONTag.getAttribute(data[key], 'id')
+                    if (!id) {
+                        id = baseURL+key+'/'
+                    }
+                    data[key] = new JSONTag.Link(id)
+                }
+            }
+        })
+    }
+    return data
+}
+
+let seen = new WeakMap()
+function deProxy(o) {
+    if (!o) {
+        return o
+    }
+    if (typeof o !== 'object') {
+        return o
+    }
+    if (seen.has(o)) {
+        return seen.get(o)
+    }
+    let result
+    if (Array.isArray(o)) {
+        result = o.map(deProxy)
+    } else if (JSONTag.isNull(o)) {
+        return o
+    } else if (JSONTag.getType(o)==='object' && o[source]) {
+        result = JSONTag.clone(o[source])
+        seen.set(o, result)
+        Object.entries(o[source]).forEach(([i,v]) => {
+            result[i] = deProxy(v)
+        })
+    } else {
+        seen.set(o, o)
+        result = o
+    }
+    return result
+}

package/src/query-worker.mjs

@@ -0,0 +1,12 @@
+import { parentPort } from 'node:worker_threads'
+import tasks from './query-worker-module.mjs'
+
+parentPort.on('message', async task => {
+	let result
+	if (tasks[task.name]) {
+		result = await tasks[task.name].call(tasks, task)
+	} else {
+		result = new Error('Unknown task '+task.name)
+	}
+	parentPort.postMessage(result)
+})