@muze-nl/simplystore 0.1.2

package/src/server.mjs

@@ -0,0 +1,323 @@
+import express from 'express'
+import fs from 'fs'
+import pointer from 'json-pointer'
+import JSONTag from '@muze-nl/jsontag'
+import {JSONPath} from 'jsonpath-plus'
+import {VM} from 'vm2'
+import _ from 'array-where-select'
+import { fileURLToPath } from 'url'
+import path from 'path'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const server    = express()
+
+function deepFreeze(obj) {
+  Object.freeze(obj)
+  Object.keys(obj).forEach(prop => {
+    if (typeof obj[prop] === 'object' && !Object.isFrozen(obj[prop])) {
+      deepFreeze(obj[prop])
+    }
+  })
+  return obj
+}
+
+function isString(s)
+{
+	return typeof s === 'string' || s instanceof String
+}
+
+function joinArgs(args) {
+	return args = args.map(arg => {
+		if (isString(arg)) {
+			return arg
+		} else {
+			return JSONTag.stringify(arg)
+		}
+	}).join(' ')
+}
+
+function connectConsole(res) {
+	return {
+		log: function(...args) {
+			res.append('X-Console-Log', joinArgs(args))
+		},
+		warning: function(...args) {
+			res.append('X-Console-Warning', joinArgs(args))
+		},
+		error: function(...args) {
+			res.append('X-Console-Error', joinArgs(args))			
+		}
+	}
+}
+
+async function main(options) {
+	if (!options) {
+		options = {}
+	}  
+  const port      = options.port || 3000;
+  const datafile  = options.datafile || 'data.jsontag'
+  const wwwroot   = options.wwwroot || __dirname+'../www'
+
+	const originalJSON = JSON
+	JSON = JSONTag // monkeypatching
+
+	console.log('loading data...')
+	let file = fs.readFileSync(datafile)
+	let dataspace;
+	let meta = {}
+	try {
+    dataspace = JSONTag.parse(file.toString(), null, meta)
+    if (typeof options.index == 'function') {
+    	options.index(dataspace, meta)
+    }
+		deepFreeze(dataspace)
+	} catch(e) {
+		console.error(e)
+		process.exit()
+	}
+
+	let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+	console.log(`data loaded (${used} MB)`);
+
+	server.get('/', (req,res) => {
+		res.send('<h1SimplyStore</h1>') //TODO: implement something nice
+	})
+
+	server.use(express.static(wwwroot))
+
+	// allow access to raw body, used to parse a query send as post body
+	server.use(express.raw({
+		type: (req) => true // parse body on all requests
+	}))
+
+	function accept(req, res, mimetypes, handler) {
+		let accept = req.accepts(mimetypes)
+		if (!accept) {
+			res.status(406)
+			res.send("<h1>406 Unacceptable</h1>\n")
+			return false
+		}
+		if (typeof handler === 'function') {
+			return handler(req, res, accept)
+		}
+		return true
+	}
+
+	function getDataSpace(req, res, dataspace) {
+		let path = req.path.substr(6); // cut '/query'
+		if (!path) {
+			path = '';
+		}
+		if (path.substring(path.length-1)==='/') {
+			//jsonpointer doesn't allow a trailing '/'
+			path = path.substring(0, path.length-1)
+		}
+		let result
+		if (path) {
+			//jsonpointer doesn't allow an empty pointer
+			try {
+				if (pointer.has(dataspace, path)) {
+					result = pointer.get(dataspace, path)
+				} else {
+					result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+				}
+			} catch(err) {
+				result = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
+			}
+		} else {
+			result = dataspace
+		}
+		return [result,path]
+	}
+	
+	let seen = new Map();
+	function countObjects(obj) {
+		if (seen.has(obj)) {
+			return 0
+		}
+		seen.set(obj, true)
+		let count = 0
+		let values = []
+		if (Array.isArray(obj)) {
+			values = obj
+			count++
+		} else if (typeof obj === 'object') {
+			if (obj instanceof String || obj instanceof Number || obj instanceof Boolean) {
+				// console.log('skipped', obj, typeof obj)
+			} else {
+				values = Object.values(obj)
+				count++
+			}
+		} else {
+			// console.log('skipped', obj, typeof obj)
+		}
+		return values
+			.filter((o) => typeof o === 'object')
+			.reduce((count, o) => count + countObjects(o), count)
+	}
+
+	server.get('/status/', (req, res, next) => 
+	{
+		seen = new Map()
+		let result = {
+			memory: Math.round(process.memoryUsage().heapUsed / 1024 / 1024)+'MB',
+			datasets: Object.keys(dataspace),
+			objects: countObjects(dataspace)
+		}
+		res.setHeader('content-type','application/json')
+		res.send(originalJSON.stringify(result, null, 4)+"\n")
+	})
+
+	server.get('/query/*', (req, res, next) => 
+	{
+		let start = Date.now()
+
+		if ( !accept(req,res,
+			['application/jsontag','application/json','text/html','text/javascript','image/*'], 
+			function(req, res, accept) {
+				switch(accept) {
+					case 'text/html':
+					case 'image/*':
+					case 'text/javascript':
+						handleWebRequest(req,res,options);
+						return false
+					break
+				}
+				return true
+			}
+		)) {
+			// done
+			return
+		}
+
+		let [result,path] = getDataSpace(req, res, dataspace)
+		if (JSONTag.getAttribute(result, 'class')==='Error') {
+			res.status(result.code)
+		}
+		result = linkReplacer(result, path+'/')
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	/**
+	 * handle queries, query is the post body
+	 */
+	server.post('/query/*',  (req, res) => {
+		let start = Date.now()
+		if ( !accept(req,res,
+			['application/jsontag','application/json']) 
+		) {
+			return
+		}
+		let error
+		let [result,path] = getDataSpace(req, res, dataspace)
+		if (result) {
+			// do the query here
+			let query = req.body.toString() // raw body through express.raw()
+			// @todo add text search: https://github.com/nextapps-de/flexsearch
+			// @todo add tree walk map/reduce/find/filter style functions
+			// @todo add arc tree dive function?
+			const vm = new VM({
+//				timeout: 1000,
+				allowAsync: false,
+				sandbox: {
+					root: dataspace,
+					data: result,
+					meta: meta,
+					_: _,
+					console: connectConsole(res),
+					JSONTag: JSONTag,
+					request: req,
+					Array: Array
+				},
+				wasm: false
+			})
+			try {
+				result = vm.run(query)
+				let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+				console.log(`(${used} MB)`);
+			} catch(err) {
+				console.log(err)
+				error = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(''+err)+',"code":422}')
+			}
+		}
+
+		if (error) {
+			res.status(error.code)
+			result = error
+		}
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	function handleWebRequest(req,res,options)
+	{
+		let path = req.path;
+		path = path.replace(/[^a-z0-9_\.\-\/]*/gi, '') // whitelist acceptable file paths
+		path = path.replace(/\.+/g, '.') // blacklist '..'
+		if (!path) {
+			path = '/'
+		}
+		if (path.substring(path.length-1)==='/') {
+			path += 'index.html'
+		}
+		const fileOptions = {
+			root: options.root || process.cwd()+'/www'
+		}
+		if (fs.existsSync(fileOptions.root+path)) {
+			res.sendFile(path, fileOptions)
+		} else {
+			res.sendFile('/index.html', fileOptions)
+		}
+	}
+
+	function linkReplacer(data, baseURL) {
+		let type = JSONTag.getType(data)
+		let attributes = JSONTag.getAttributes(data)
+		if (Array.isArray(data)) {
+			data = data.map((entry,index) => {
+				return linkReplacer(data[index], baseURL+index+'/')
+			})
+		} else if (type === 'link') {
+			// do nothing
+		} else if (data && typeof data === 'object') {
+			data = JSONTag.clone(data)
+			Object.keys(data).forEach(key => {
+				if (Array.isArray(data[key])) {
+					data[key] = new JSONTag.Link(baseURL+key+'/')
+				} else if (typeof data[key] === 'object') {
+					if (JSONTag.getType(data[key])!=='link') {
+						let id=JSONTag.getAttribute(data[key], 'id')
+						if (!id) {
+							id = baseURL+key+'/'
+						}
+						data[key] = new JSONTag.Link(id)
+					}
+				}
+			})
+		}
+		return data
+	}
+
+	server.listen(port, () => {
+		console.log('SimplyStore listening on port '+port)
+	})
+
+}
+
+server.run = main
+export default server

package/src/triplestore.mjs

@@ -0,0 +1,182 @@
+import JSONTag from '@muze-nl/jsontag'
+
+export default class TripleStore {
+
+	triples=[]
+	#index
+	#availablePredicates
+
+	constructor(jsontag) {
+		this.#index = {
+			subject: new WeakMap(),
+			predicate: {},
+			object: new WeakMap(),
+			value: {}
+		}
+		this.#availablePredicates = new Set()
+		this.makeTripleStore(jsontag)
+		this.createIndex()
+		console.log(this.#availablePredicates)
+	}
+
+    makeTripleStore(jsontag, parent=null, predicate=null) {
+    	let seen = new Map()
+    	let addTriples = (jsontag, parent=null, predicate=null) => {
+    		if (jsontag && typeof jsontag === 'object') {
+	    		if (seen.has(jsontag)) {
+	    			return
+	    		}
+	    		seen.set(jsontag, true)
+	    	}
+			if (Array.isArray(jsontag)) {
+				jsontag.forEach(e => {
+					addTriples(e, parent, predicate)
+				})
+			} else if (typeof jsontag === 'object') {
+				if (parent) {
+					this.triples.push([parent, predicate, jsontag ])
+					this.#availablePredicates.add(predicate)
+				}
+				let tagname = JSONTag.getType(jsontag)
+				this.triples.push([jsontag, '@tag', tagname])
+				this.#availablePredicates.add('@tag')
+				let attributes = JSONTag.getAttributes(jsontag)
+				Object.keys(attributes).forEach(a => {
+					this.triples.push([jsontag, '@attr/'+a, attributes[a]])
+					this.#availablePredicates.add('@attr/'+a)
+				})
+				// handle String, Number, Boolean seperately.. don't forEach there
+				if (jsontag && JSONTag.getType(jsontag)==='object') {
+					Object.keys(jsontag).forEach(p => {
+						addTriples(jsontag[p], jsontag, p);
+					})
+				}
+			} else {
+	/*
+				switch(typeof jsontag) {
+					case 'number':
+						jsontag = new Number(jsontag)
+					break
+					case 'boolean':
+						jsontag = new Boolean(jsontag)
+					break
+					case 'string':
+						jsontag = new String(jsontag)
+					break
+					default:
+						return
+					break
+				}
+	*/
+				this.triples.push([parent, predicate, jsontag]) 
+				this.#availablePredicates.add(predicate)
+			}
+		}
+		addTriples(jsontag)
+	}
+
+	query({ find, where }) {
+		return this.queryWhere(where).map(context => this.actualize(context, find))
+	}
+
+	actualize(context, find) {
+		return find.map( findPart => {
+			return this.isVariable(findPart) ? context[findPart] : findPart
+		})
+	}
+
+	queryWhere(patterns) {
+		return patterns.reduce( (contexts, pattern) => {
+			return contexts.flatMap( context => this.querySingle(pattern, context))
+		}, [{}])
+	}
+
+	querySingle(pattern, context) {
+		return this.relevantTriples(pattern)
+		.map(triple => this.matchPattern(pattern, triple, context))
+		.filter( x => x)
+	}
+
+	matchPattern(pattern, triple, context) {
+		return pattern.reduce((context, patternPart, idx) => {
+			const triplePart = triple[idx]
+			return this.matchPart(patternPart, triplePart, context)
+		}, context)
+	}
+
+	matchPart(patternPart, triplePart, context) {
+		if (!context) {
+			return null
+		}
+		if (this.isVariable(patternPart)) {
+			return this.matchVariable(patternPart, triplePart, context)
+		}
+		if (patternPart instanceof Function) {
+			return (patternPart(triplePart) ? context : null)
+		}
+		return patternPart == triplePart ? context : null
+	}
+
+	isVariable(x) {
+		return typeof x === 'string' && x.startsWith('?')
+	}
+
+	matchVariable(variable, triplePart, context) {
+		if (context.hasOwnProperty(variable)) {
+			const bound = context[variable]
+			return this.matchPart(bound, triplePart, context)
+		}
+		return { ...context, [variable]: triplePart }
+	}
+
+	relevantTriples(pattern) {
+		const [subject, predicate, object] = pattern
+		if (!this.#availablePredicates.has(predicate)) {
+			throw new Error('TripleStore does not contain predicate '+predicate)
+		}
+		if (!this.isVariable(subject)) {
+		    return this.#index.subject[subject] || []
+		}
+		if (!this.isVariable(predicate)) {
+		    return this.#index.predicate[predicate] || []
+		}
+		if (!this.isVariable(object)) {
+			if (typeof object === 'object') {
+			    return this.#index.object[object] || []
+			} else {
+				return this.#index.value[object] || []
+			}
+		}
+		return this.triples
+	}
+
+	createIndex() {
+		this.indexBy(0, this.#index.subject)
+		this.indexBy(1, this.#index.predicate)
+		this.indexBy(2, this.#index.object, this.#index.value)
+	}
+
+	indexBy(idx, index1, index2=null) {
+		let count = 0;
+		return this.triples.forEach((triple) => {
+//			let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+//			console.log(idx, count++, used)
+			const k = triple[idx]
+			if (typeof k === 'object') {
+				index1[k] = index1[k] || []
+				index1[k].push(triple)
+				if (index2 && JSONTag.getType(k)!=='object' && JSONTag.getType(k)!=='array') {
+					let v = k.valueOf()
+					index2[v] = index2[v] || []
+					index2[v].push(triple)
+				}
+			} else if (index2) {
+				index2[k] = index2[k] || []
+				index2[k].push(triple)
+			} else {
+				index1[k] = index1[k] || []
+				index1[k].push(triple)
+			}
+		})
+	}
+}