@muze-nl/simplystore 0.1.2 → 0.1.5

package/default.jsontag

@@ -0,0 +1,18 @@
+{
+	"persons": [
+		<object id="john" class="Person">{
+			"name": "John",
+			"dob": <date>"1972-09-20",
+			"foaf": [
+				<link>"jane"
+			]
+		},
+		<object id="jane" class="Person">{
+			"name": "Jane",
+			"dob": <date>"1986-01-01",
+			"foaf": [
+				<link>"john"
+			]
+		}
+	]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@muze-nl/simplystore",
-	"version": "0.1.2",
+	"version": "0.1.5",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {

package/package.json~

@@ -1,6 +1,6 @@
 {
-	"name": "@muze-nl/SimplyStore",
-	"version": "0.1.2",
+	"name": "@muze-nl/simplystore",
+	"version": "0.1.4",
 	"main": "src/server.mjs",
 	"type": "module",
 	"scripts": {

package/simplystore-talk.md

@@ -0,0 +1,45 @@
+simplystore
+- logo
+- website
+- talk
+	focus op max 3 problemen/oplossingen
+		linked data complexiteit - tussenstap met jsontag
+		minder moving parts - code naar data brengen ipv andersom
+
+---
+
+SimplyStore
+
+All remote API's suck, can we make it better?
+
+Why do we need remote API's?
+
+Large datasets
+Collaboration
+Control of centralized resources
+
+What are the problems with remote API's
+
+Local resources and data can be managed with normal code flow
+Remote API's require breaking out, with usually RPC calls
+That means - asynchronous calls, converting data from/to a stringified format, limited to whatever the API provides
+
+RPC means network, network access is slow
+API's provide limited semantics, usually HTTP endpoints with JSON, if you are lucky GraphQL
+So you usually end up fetching too much data, to filter it down locally. Or you do many batches of smaller fetches. Both are always too slow.
+
+Finally, API's change. So all clients must change. Each API is unique, so all client code is unique. Any change in a single API blossoms into requiring changes in many clients. Schema changes are the worst. Versioning helps, but keep old versions running! 
+
+Sometimes changes are so large that older versions cannog be kept. This is a shame.
+
+OpenAPI (Swagger) is not a solution. The semantics of the data and schema changes aren't tackled with OpenAPI specifications. 
+
+What technology exists that promises a solution?
+
+AI. done.
+
+Ok, so not AI. How about Linked Data?
+
+
+
+

package/src/DEADJOE

@@ -0,0 +1,3 @@
+
+*** These modified files were found in JOE when it aborted on Thu Mar 23 15:39:53 2023
+*** JOE was aborted because the terminal closed

package/src/editor.mjs~

@@ -0,0 +1,7 @@
+import {EditorView, basicSetup} from "codemirror"
+import {javascript} from "@codemirror/lang-javascript"
+
+let codeEditor = new EditorView({
+  extensions: [basicSetup, javascript()],
+  parent: document.body
+})

package/src/main.mjs~

@@ -0,0 +1,301 @@
+import express from 'express'
+import fs from 'fs'
+import pointer from 'json-pointer'
+import JSONTag from '@muze-nl/jsontag'
+import {JSONPath} from 'jsonpath-plus'
+import jsonExt from '@discoveryjs/json-ext'
+//import TripleStore from './triplestore.mjs'
+import {VM} from 'vm2'
+
+const server    = express()
+const port      = process.env.NODE_PORT || 3000;
+
+const datafile  = process.env.DATAFILE || 'data.jsontag'
+
+server.use(express.static(process.cwd()+'/www'))
+
+async function main() {
+
+	const originalJSON = JSON
+	JSON = JSONTag // monkeypatching
+
+	console.log('loading data...')
+//	let file = fs.readFileSync(datafile);
+//	let dataspace = JSONTag.parse(file.toString());
+	let reader = fs.createReadStream(datafile);
+	let dataspace = await JSONTag.parseStream(reader);
+
+	console.log('indexing data...')
+//	let tripleStore = new TripleStore(dataspace)
+
+	let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+	console.log(`data loaded (${used} MB)`);
+
+	// allow access to raw body, used to parse a query send as post body
+	server.use(express.raw({
+		type: (req) => true // parse body on all requests
+	}))
+
+	server.get('/query/*', (req, res, next) => 
+	{
+		let start = Date.now()
+		let accept = req.accepts(['application/jsontag','application/json','text/html','text/javascript','image/*'])
+		if (!accept) {
+			res.status(406)
+			res.send("<h1>406 Unacceptable</h1>\n")
+			return
+		}
+		switch(accept) {
+			case 'text/html':
+			case 'image/*':
+			case 'text/javascript':
+				handleWebRequest(req,res);
+				return 
+			break
+		}
+
+		let path = req.path.substr(6); // cut '/query'
+		if (!path) {
+			path = '';
+		}
+		if (path.substring(path.length-1)==='/') {
+			//jsonpointer doesn't allow a trailing '/'
+			path = path.substring(0, path.length-1)
+		}
+		let result
+		if (path) {
+			//jsonpointer doesn't allow an empty pointer
+			try {
+				if (pointer.has(dataspace, path)) {
+					result = pointer.get(dataspace, path)
+				} else {
+					result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+				}
+			} catch(err) {
+				result = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
+			}
+		} else {
+			result = dataspace
+		}
+		if (JSONTag.getAttribute(result, 'class')==='Error') {
+			res.status(result.code)
+		}
+		result = linkReplacer(result, path+'/')
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	/**
+	 * handle queries, query is the post body
+	 */
+	server.post('/query/*',  (req, res) => {
+		let start = Date.now()
+		function parseParams(paramsString) {
+			let result = {}
+			let lineRe = /^(.*)$/m
+			let parseRe = /^\s*([a-z].*?)?\s*(\{)?\s*$/i
+			let line = lineRe.exec(paramsString).pop()
+			let full, prop, recurse
+			do {
+				paramsString = paramsString.substring(line.length+1)
+				let parsed = parseRe.exec(line)
+				if (parsed) {
+					[full,prop,recurse] = parsed
+				}
+				if (recurse) {
+					[ result[prop], paramsString ] = parseParams(paramsString)
+				} else if (prop) {
+					result[prop] = ''
+				} else if (/\}/.exec(line)) {
+					return [ result, paramsString ]
+				}
+				line = lineRe.exec(paramsString).pop()
+			} while(line)
+			return [ result, '' ]
+		}
+
+		function filterProperties(paramsOb) {
+			return function(object) {
+				let result = {}
+				let obType = JSONTag.getType(object)
+				JSONTag.setType(result, obType)
+				JSONTag.setAttributes(result, JSONTag.getAttributes(object))
+				Object.entries(paramsOb).forEach(([key,value]) =>{
+					let alias, queryResult
+					[alias,key] = key.split(':',2).map(p => p.trim())
+					if (!key) {
+						key = alias
+					}
+					if (key[0]==='$') {
+						queryResult = JSONPath({path:key, json:object, flatten: true})
+					} else {
+						queryResult = object[key]
+					}
+					if (!value) {
+						// @TODO: just set the entire value for now, should run linkReplacer here
+						// this is better solved after forcing all objects to have an id attribute with a unique id
+						result[alias] = queryResult
+					} else {
+						result[alias] = filterProperties(value)(queryResult)
+					}
+				})
+				return result;
+			}
+		}
+
+		let accept = req.accepts(['application/jsontag','application/json','text/html','text/javascript','image/*'])
+		if (!accept) {
+			res.status(406)
+			res.send("<h1>406 Unacceptable</h1>\n")
+			return
+		}
+		let path = req.path.substring(6); // cut '/query'
+		if (!path) {
+			path = '';
+		}
+		if (path.substring(path.length-1)==='/') {
+			//jsonpointer doesn't allow a trailing '/'
+			path = path.substring(0, path.length-1)
+		}
+		let error,result
+		if (path) {
+			//jsonpointer doesn't allow an empty pointer
+			try {
+				if (pointer.has(dataspace, path)) {
+					result = pointer.get(dataspace, path)
+				} else {
+					error = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+				}
+			} catch(err) {
+				error = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
+			}
+		} else {
+			result = dataspace
+		}
+
+		if (result) {
+			// do the query here
+			let query = req.body.toString() // raw body through express.raw()
+			console.log(query)
+			// @todo add text search: https://github.com/nextapps-de/flexsearch
+			// @todo add tree walk map/reduce/find/filter style functions
+			// @todo add arc tree dive function?
+			const vm = new VM({
+//				timeout: 1000,
+				allowAsync: false,
+				sandbox: {
+//					query: function(params) {
+//						return tripleStore.query(params)
+//					}
+				},
+				wasm: false
+			})
+			vm.freeze(result, 'data') // adds immutable result dataspace to sandbox as data
+			try {
+				result = vm.run(query)
+				let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+				console.log(`(${used} MB)`);
+			} catch(err) {
+				console.log(err)
+				error = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(''+err)+',"code":422}')
+			}
+		}
+
+		if (error) {
+			res.status(error.code)
+			result = error
+		}
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	server.get('/', (req,res) => {
+		res.send('<h1>JSONTag REST+ server</h1>')
+	})
+
+	function handleWebRequest(req,res) 
+	{
+		let path = req.path;
+		path = path.replace(/[^a-z0-9_\.\-\/]*/gi, '') // whitelist acceptable file paths
+		path = path.replace(/\.+/g, '.') // blacklist '..'
+		if (!path) {
+			path = '/'
+		}
+		if (path.substring(path.length-1)==='/') {
+			path += 'index.html'
+		}
+		const options = {
+			root: process.cwd()+'/www'
+		}
+		if (fs.existsSync(options.root+path)) {
+			res.sendFile(path, options)
+		} else {
+			res.sendFile('/index.html', options)
+		}
+	}
+
+	function linkReplacer(data, baseURL) {
+		let type = JSONTag.getType(data)
+		let attributes = JSONTag.getAttributes(data)
+		if (Array.isArray(data)) {
+			data = data.map((entry,index) => {
+				return linkReplacer(data[index], baseURL+index+'/')
+			})
+		} else if (type === 'link') {
+			// do nothing
+		} else if (data && typeof data === 'object') {
+			data = JSONTag.clone(data)
+			Object.keys(data).forEach(key => {
+				if (Array.isArray(data[key])) {
+					data[key] = data[key].map((e,i) => {
+						if (e && typeof e === 'object') {
+							let id=JSONTag.getAttribute(e, 'id')
+							if (!id) {
+								id = baseURL+key+'/'+i+'/'
+							} else {
+								id = '#'+id
+							}
+							return new JSONTag.Link(id)
+						}
+						return e
+					})
+				} else if (typeof data[key] === 'object') {
+					let id=JSONTag.getAttribute(data[key], 'id')
+					if (!id) {
+						id = baseURL+key+'/'
+					} else {
+						id = '#'+id
+					}
+					data[key] = new JSONTag.Link(id)
+				}
+			})
+		}
+		return data
+	}
+
+	function isString(s)
+	{
+		return typeof s === 'string' || s instanceof String
+	}
+	server.listen(port, () => 
+	{
+		console.log('JSONTag REST server listening on port '+port)
+	})
+
+}
+
+main()

package/src/server.js

@@ -0,0 +1,278 @@
+import express from 'express'
+import fs from 'fs'
+import pointer from 'json-pointer'
+import JSONTag from '@muze-nl/jsontag'
+import {JSONPath} from 'jsonpath-plus'
+import {VM} from 'vm2'
+import _ from 'array-where-select'
+
+const server    = express()
+
+server.use(express.static(process.cwd()+'/www'))
+
+function deepFreeze(obj) {
+  Object.freeze(obj)
+  Object.keys(obj).forEach(prop => {
+    if (typeof obj[prop] === 'object' && !Object.isFrozen(obj[prop])) {
+      deepFreeze(obj[prop])
+    }
+  })
+  return obj
+}
+
+function joinArgs(args) {
+	return args = args.map(arg => {
+		if (isString(arg)) {
+			return arg
+		} else {
+			return JSONTag.stringify(arg)
+		}
+	}).join(' ')
+}
+
+function connectConsole(res) {
+	return {
+		log: function(...args) {
+			res.append('X-Console-Log', joinArgs(args))
+		},
+		warning: function(...args) {
+			res.append('X-Console-Warning', joinArgs(args))
+		},
+		error: function(...args) {
+			res.append('X-Console-Error', joinArgs(args))			
+		}
+	}
+}
+
+async function main(options) {
+  
+  const port      = options.port || 3000;
+  const datafile  = options.datafile || 'data.jsontag'
+
+	const originalJSON = JSON
+	JSON = JSONTag // monkeypatching
+
+	console.log('loading data...')
+	let file = fs.readFileSync(datafile)
+	let dataspace;
+	let meta = {}
+	try {
+    dataspace = JSONTag.parse(file.toString(), null, meta)
+    if (typeof options.index == 'function') {
+    	options.index(dataspace, meta)
+    }
+		deepFreeze(dataspace)
+	} catch(e) {
+		console.error(e)
+		process.exit()
+	}
+
+	let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+	console.log(`data loaded (${used} MB)`);
+
+	// allow access to raw body, used to parse a query send as post body
+	server.use(express.raw({
+		type: (req) => true // parse body on all requests
+	}))
+
+	function accept(req, res, mimetypes, handler) {
+		let accept = req.accepts(mimetypes)
+		if (!accept) {
+			res.status(406)
+			res.send("<h1>406 Unacceptable</h1>\n")
+			return false
+		}
+		if (typeof handler === 'function') {
+			return handler(req, res, accept)
+		}
+		return true
+	}
+
+	function getDataSpace(req, res, dataspace) {
+		let path = req.path.substr(6); // cut '/query'
+		if (!path) {
+			path = '';
+		}
+		if (path.substring(path.length-1)==='/') {
+			//jsonpointer doesn't allow a trailing '/'
+			path = path.substring(0, path.length-1)
+		}
+		let result
+		if (path) {
+			//jsonpointer doesn't allow an empty pointer
+			try {
+				if (pointer.has(dataspace, path)) {
+					result = pointer.get(dataspace, path)
+				} else {
+					result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+				}
+			} catch(err) {
+				result = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(err.message)+', "code":500}')
+			}
+		} else {
+			result = dataspace
+		}
+		return result
+	}
+	
+	server.get('/query/*', (req, res, next) => 
+	{
+		let start = Date.now()
+
+		if ( !accept(req,res,
+			['application/jsontag','application/json','text/html','text/javascript','image/*'], 
+			function(req, res, accept) {
+				switch(accept) {
+					case 'text/html':
+					case 'image/*':
+					case 'text/javascript':
+						handleWebRequest(req,res,options);
+						return false
+					break
+				}
+				return true
+			}
+		)) {
+			// done
+			return
+		}
+
+		result = getDataSpace(req, res, dataspace)
+		if (JSONTag.getAttribute(result, 'class')==='Error') {
+			res.status(result.code)
+		}
+		result = linkReplacer(result, path+'/')
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	/**
+	 * handle queries, query is the post body
+	 */
+	server.post('/query/*',  (req, res) => {
+		let start = Date.now()
+		if ( !accept(req,res,
+			['application/jsontag','application/json']) {
+			return
+		}
+
+		result = getDataSpace(req, res, dataspace)
+		if (result) {
+			// do the query here
+			let query = req.body.toString() // raw body through express.raw()
+			// @todo add text search: https://github.com/nextapps-de/flexsearch
+			// @todo add tree walk map/reduce/find/filter style functions
+			// @todo add arc tree dive function?
+			const vm = new VM({
+//				timeout: 1000,
+				allowAsync: false,
+				sandbox: {
+					root: dataspace,
+					data: result,
+					meta: meta,
+					_: _,
+					console: connectConsole(res),
+					JSONTag: JSONTag,
+					request: req
+				},
+				wasm: false
+			})
+			try {
+				result = vm.run(query)
+				let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+				console.log(`(${used} MB)`);
+			} catch(err) {
+				console.log(err)
+				error = JSONTag.parse('<object class="Error">{"message":'+originalJSON.stringify(''+err)+',"code":422}')
+			}
+		}
+
+		if (error) {
+			res.status(error.code)
+			result = error
+		}
+		if (req.accepts('application/jsontag')) {
+			res.setHeader('content-type','application/jsontag+json')
+			res.send(JSONTag.stringify(result, null, 4)+"\n")
+		} else {
+			res.setHeader('content-type','application/json')
+			res.send(originalJSON.stringify(result, null, 4)+"\n")
+		}
+		let end = Date.now()
+		console.log(path, (end-start), process.memoryUsage())
+	})
+
+	server.get('/', (req,res) => {
+		res.send('<h1>JSONTag REST+ server</h1>') //TODO: implement something nice
+	})
+
+	function handleWebRequest(req,res,options) 
+	{
+		let path = req.path;
+		path = path.replace(/[^a-z0-9_\.\-\/]*/gi, '') // whitelist acceptable file paths
+		path = path.replace(/\.+/g, '.') // blacklist '..'
+		if (!path) {
+			path = '/'
+		}
+		if (path.substring(path.length-1)==='/') {
+			path += 'index.html'
+		}
+		const fileOptions = {
+			root: options.root || process.cwd()+'/www'
+		}
+		if (fs.existsSync(fileOptions.root+path)) {
+			res.sendFile(path, fileOptions)
+		} else {
+			res.sendFile('/index.html', fileOptions)
+		}
+	}
+
+	function linkReplacer(data, baseURL) {
+		let type = JSONTag.getType(data)
+		let attributes = JSONTag.getAttributes(data)
+		if (Array.isArray(data)) {
+			data = data.map((entry,index) => {
+				return linkReplacer(data[index], baseURL+index+'/')
+			})
+		} else if (type === 'link') {
+			// do nothing
+		} else if (data && typeof data === 'object') {
+			data = JSONTag.clone(data)
+			Object.keys(data).forEach(key => {
+				if (Array.isArray(data[key])) {
+					data[key] = new JSONTag.Link(baseURL+key+'/')
+				} else if (typeof data[key] === 'object') {
+					if (JSONTag.getType(data[key])!=='link') {
+						let id=JSONTag.getAttribute(data[key], 'id')
+						if (!id) {
+							id = baseURL+key+'/'
+						}
+						data[key] = new JSONTag.Link(id)
+					}
+				}
+			})
+		}
+		return data
+	}
+
+	function isString(s)
+	{
+		return typeof s === 'string' || s instanceof String
+	}
+
+	server.listen(port, () => 
+	{
+		console.log('JSONTag REST server listening on port '+port)
+	})
+
+}
+
+server.run = main
+export default server