@muze-nl/simplystore 0.3.2 → 0.3.5

package/src/worker-query.mjs

@@ -0,0 +1,146 @@
+import JSONTag from "@muze-nl/jsontag"
+import pointer from 'json-pointer'
+import {_,from,not,anyOf,allOf} from 'array-where-select'
+import {deepFreeze} from './util.mjs'
+import {VM} from 'vm2'
+
+let dataspace, meta = {};
+
+export function setDataspace(d, m) {
+    dataspace = d
+    if(m) {
+        meta = m
+    }
+}
+
+export function getDataSpace(path, dataspace) {
+    if (path.substring(path.length-1)==='/') {
+        //jsonpointer doesn't allow a trailing '/'
+        path = path.substring(0, path.length-1)
+    }
+    let result
+    if (path) {
+        //jsonpointer doesn't allow an empty pointer
+        try {
+            if (pointer.has(dataspace, path)) {
+                result = pointer.get(dataspace, path)
+            } else {
+                result = JSONTag.parse('<object class="Error">{"message":"Not found", "code":404}')
+            }
+        } catch(err) {
+            result = JSONTag.parse('<object class="Error">{"message":'+JSON.stringify(err.message)+', "code":500}')
+        }
+    } else {
+        result = dataspace
+    }
+    return [result,path]
+}
+
+export function linkReplacer(data, baseURL) {
+    let type = JSONTag.getType(data)
+    let attributes = JSONTag.getAttributes(data)
+    if (Array.isArray(data)) {
+        data = data.map((entry,index) => {
+            return linkReplacer(data[index], baseURL+index+'/')
+        })
+    } else if (type === 'link') {
+        // do nothing
+    } else if (data && typeof data === 'object') {
+        data = JSONTag.clone(data)
+        Object.keys(data).forEach(key => {
+            if (Array.isArray(data[key])) {
+                data[key] = new JSONTag.Link(baseURL+key+'/')
+            } else if (data[key] && typeof data[key] === 'object') {
+                if (JSONTag.getType(data[key])!=='link') {
+                    let id=JSONTag.getAttribute(data[key], 'id')
+                    if (!id) {
+                        id = baseURL+key+'/'
+                    }
+                    data[key] = new JSONTag.Link(id)
+                }
+            }
+        })
+    }
+    return data
+}
+
+//@TODO: emit console events that server.mjs picks up
+function connectConsole(res) {
+    return {
+        log: function(...args) {
+//            res.append('X-Console-Log', joinArgs(args))
+        },
+        warning: function(...args) {
+//            res.append('X-Console-Warning', joinArgs(args))
+        },
+        error: function(...args) {
+//            res.append('X-Console-Error', joinArgs(args))            
+        }
+    }
+}
+
+export async function initialize(jsontag, m) {
+    if (!jsontag) { throw new Error('missing jsontag parameter')}
+	dataspace = jsontag
+    meta = m
+    deepFreeze(dataspace)
+    return true
+}
+
+export function runQuery({pointer, request, query}) {
+    if (!pointer) { throw new Error('missing pointer parameter')}
+    if (!request) { throw new Error('missing request parameter')}
+	console.log('query',pointer)
+	let response = {
+        jsontag: request.jsontag
+    }
+    let [result,path] = getDataSpace(pointer, dataspace)
+
+    if (query) {
+        // @todo add text search: https://github.com/nextapps-de/flexsearch
+        // @todo add tree walk map/reduce/find/filter style functions
+        // @todo add arc tree dive function?
+        // @todo replace VM with V8 isolate
+        const vm = new VM({
+            timeout: 1000,
+            allowAsync: false,
+            sandbox: {
+                root: dataspace, //@TODO: if we don't pass the root, we can later shard
+                data: result,
+                meta,
+                _,
+                from,
+                not,
+                anyOf,
+                allOf,
+//                    console: connectConsole(res),
+                JSONTag,
+                request
+            },
+            wasm: false
+        })
+        try {
+            result = vm.run(query)
+            let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
+            console.log(`(${used} MB)`);
+        } catch(err) {
+            console.log(err)
+            response.code = 422;
+            if (request.jsontag) {
+            	response.body = '<object class="Error">{"message":'+JSON.stringify(''+err)+',"code":422}'
+            } else {
+            	response.body = JSON.stringify({message:err, code: 422})
+            }
+        }
+    } else {
+        result = linkReplacer(result, path+'/')
+    }
+    if (!response.code) {
+        if (response.jsontag) {
+        	response.body = JSONTag.stringify(result)
+        } else {
+        	response.body = JSON.stringify(result)
+        }
+    }
+    return response
+}

package/test/produce.mjs

@@ -0,0 +1,79 @@
+import tap from 'tap'
+import fs from 'fs'
+import JSONTag from '@muze-nl/jsontag'
+import {produce,index} from '../src/produce.mjs'
+import {deepFreeze} from '../src/util.mjs'
+
+let data = deepFreeze(JSONTag.parse(fs.readFileSync('./test/test.jsontag','utf-8')))
+
+tap.test('data is frozen', t => {
+	t.throws(() => {
+		data.persons.foo = 'bar'
+	})
+	t.notHas(data.persons, {foo:'bar'})
+	t.end()
+})
+
+tap.test('produce can create new data', t => {
+	let newData = produce(data, (draft) => {
+		draft.persons.foo = 'bar'
+	})
+	t.has(newData.persons, {foo:'bar'})
+	t.end()
+})
+
+tap.test('produce does not change base data', t => {
+	let newData = produce(data, (draft) => {
+		console.log('Persons draft',draft.persons, Object.isFrozen(draft.persons))
+		draft.persons.foo = 'bar'
+	})
+	t.notHas(data.persons, {foo:'bar'})
+	t.end()	
+})
+
+tap.test('produce handles array access', t => {
+	let newData = produce(data, (draft) => {
+		draft.persons[0].name = 'Jan'
+	})
+	t.equal(newData.persons[0].name, 'Jan')
+	t.end()
+})
+
+tap.test('produce handles array functions', t => {
+	let newData = produce(data, (draft) => {
+		draft.persons.push({
+			name: 'Jan'
+		})
+	})
+	t.equal(newData.persons[2].name, 'Jan')
+	t.same(data.persons[2],null)
+	t.end()
+})
+
+tap.test('produce can use array.indexOf inside', t => {
+	let newData = produce(data, (draft) => {
+		let p = draft.persons[1] // this returns a proxy
+		let i = draft.persons.indexOf(p) // this is passed on to the baseState/clone, which has values without proxy
+		t.equal(i,1) // so this no longer fails, as indexOf automatically calls getRealValue on all params
+	})
+	t.end()
+})
+
+tap.test('produce does not alter unaccessed objects', t => {
+	let newData = produce(data, (draft) => {
+		draft.persons.foo = 'bar'
+	})
+	t.equal(data.persons[0],newData.persons[0])
+	t.equal(data.persons[1],newData.persons[1])
+	t.notEqual(data.persons,newData.persons)
+	t.end()
+})
+
+tap.test('proxies get re-used', t => {
+	let newData = produce(data, (draft) => {
+		let p1 = draft.persons[0]
+		let p2 = draft.persons[0]
+		t.equal(p1,p2)
+	})
+	t.end()
+})

package/test/share.mjs

@@ -0,0 +1,18 @@
+import tap from 'tap'
+import {share} from '../src/share.mjs'
+import JSONTag from '@muze-nl/jsontag'
+import fs from 'fs'
+
+let data = JSONTag.parse(fs.readFileSync('./test/test.jsontag','utf-8'))
+
+tap.test('create shared array buffer', t => {
+	let shared = share(data)
+	console.log(shared)
+	let decoder = new TextDecoder()
+	let uint8buffer = new Uint8Array(shared.buffer)
+	let str = decoder.decode(uint8buffer)
+	console.log('string',str)
+
+	t.end()
+})
+

package/test/test.jsontag

@@ -0,0 +1,20 @@
+{
+	"persons": [
+		<object id="john" class="Person">{
+			"name": "John",
+			"lastName": "Doe",
+			"dob": <date>"1972-09-20",
+			"foaf": [
+				<link>"jane"
+			]
+		},
+		<object id="jane" class="Person">{
+			"name": "Jane",
+			"lastName": "Doe",
+			"dob": <date>"1986-01-01",
+			"foaf": [
+				<link>"john"
+			]
+		}
+	]
+}

package/www/codemirror/keymap/vim.js

@@ -5966,9 +5966,9 @@ function initVim$1(CodeMirror) {
   return vimApi;
 }
 
-function initVim(CodeMirror5) {
-  CodeMirror5.Vim = initVim$1(CodeMirror5);
-  return CodeMirror5.Vim;
+function initVim(CodeMirror5) {
+  CodeMirror5.Vim = initVim$1(CodeMirror5);
+  return CodeMirror5.Vim;
 }
 
 

package/design/access-management.md

@@ -1,25 +0,0 @@
-# access management (grants/rights/roles)
-
-## Read grants
-
-You can easily create a read access check that just checks the query/ endpoint, but this is not interesting.
-Much more interesting is how to implement granular read access rights on subtrees of the dataset.
-
-Ariadne has a mechanism that has been succesfully used since 1998. It allows you to defined grant strings, like 'read', 'edit', etc. and configure them on a path inside a tree of data. Each entity below that path will automatically inherit the grant.
-
-The grant is then checked for each interaction with the data. Because all interactions in Ariadne are through templates, which can be custom made, the grants can also be custom strings. This way you can grow the access management system to your own needs.
-
-However, this means that all read access must come through the data tree. Earlier we defined a /uuid/ endpoint which would give direct access to any object. This breaks this paradigm. Objects can be linked in multiple locations in the dataset. To check for read grants, you must find all valid paths to an object and check if any of them allow the user to 'read' the object. This is potentially very costly.
-
-The easiest solution is to drop the /uuid endpoint, unless all data is publically readable. Each object can still have a uuid, but it is no longer a url that points to the objects contents.
-
-Another problem is the changing nature of the JSON Pointer path in the URL. If an object is part of an array, and another object earlier (with smalled index) is removed, the URL for this object changes. Its index is lowered. So you cannot assign grants on a JSON Pointer path, containing an array index, with any hope of the grants staying in the correct spot.
-
-If you instead assign grants to objects directly, the grants will correctly move when an array is updated. However this opens up the possibility that grants appear in multiple places, if the object is linked in multiple places. This should not be a problem though, since the subtree of objects is the same as long as the object is the same.
-
-This opens up the possibility of storing the grants in an attribute on the JSONTag tag of the object. e.g.
-
-<object class="Site" grants="user1: read edit, user2: read edit delete">{ ... }
-
-Here we need to carefully consider how to treat links. Do we need read access to follow a link, or is that implied?
-

package/design/acid.md

@@ -1,31 +0,0 @@
-# JSONTAG REST Server - Commands / CQRS
-
-Why the need for CQRS (Command Query Responsibility Segregation?)
-
-The design of the server is purposely simple. It does away with a seperate database system, instead reading all data in memory, giving access to that data to queries written in javascript, using native javascript objects. This does away with the relational-object impedance mismatch. It does away with SQL and ORM solutions.
-
-However, databases have the nice property that they are ACID compliant. We don't want to lose that. ACID stands for 'Atomicity', 'Consistency', 'Isolation' and 'Durability'. Our server should exhibit these same properties. Isolation is handled by handling each query in a seperate VM, using immutable data. But immutable data also means we need a different mechanism to update data, other than plain javascript.
-
-Why the need for immutable data? This is tied to the ACID requirements. If you have multiple processes working with the same, shared, data, there is a good chance of creating inconsistent data. One process changes the data in some way, while another process tries to change the exact same data. One of these processes will 'win', potentially undoing the other process' change. Imagine you have a shop with an inventory. If product X has 1 item in the inventory, and there are two requests to buy that product being handled by two different processes at the same time, you could end up selling the same item twice.
-
-There are ways around this, using locks and mutexes, but these are notoriously difficult to get right. 
-
-Just using immutable data alone won't solve this problem. Both processes will still see that there is 1 product X in store. But if we only allow updates / changes, through a single sequential process, and we allow for the possibility that a change request can fail, we can solve this problem much easier.
-
-So both processes see that there is one item left in inventory. Both processes now create a command (buyProduct(X)) and send it to the command queue. They both get a unique ID as a result, and they can listen for an update on that ID. But there is only a single command handler, which handles commands on a First In First Out basis. The first buyProduct(X) command succeeds, and the inventory is decreased by 1. The next buyProduct(X) command fails its precondition, the inventory is 0. So it fails. Both processes get an update, one is a success, the other is a failure.
-
-Now as the data is immutable, this is not entirely correct. In fact there is a difference between the client process calling the server, and the server process which only allows you to query the data. The server process can't create a command or listen for updates, only the client can. So when the client receives a success or failure resutl, asynchronously, it can send a new query to the server. The server in the mean time creates a new query handler, with the updated data after the changes from the last commands.
-
-Because changes are only applied using the single process command handler, Atomicity is preserved. Each command is an atomic update, it either succeeds of fails. If it fails, none of the updates are kept, since the immutable data structure root is not updated.
-
-Consistency is preserved. Each query request sees a concistent dataset, it may be slightly out of date, but it is internally consistent. Later queries will see a more up to date version of the data, so in the context of updates, the data is eventually consistent.
-
-Isolation is preserved. Each query runs in its own VM, with its own access to the immutable data. Each command is run seperate from queries and custom javascript. 
-
-Durability is preserved. Each command is first written to a log, then the unique ID of the command is returned. The whole dataset is backup to disk once in a while, with the last command ID that has been processed. A backup can be restored, then all commands after the last processes one can be processed again.
-
-The query handlers can run parallel, since they only have access to immutable data. In fact, each query handler process can use the exact same shared memory, keeping memory usage low.
-
-There is a potential problem that a client sends a command, which the server enters into the command log, but before the command log id can be sent back to the client, the server dies. In that case the client cannot know if the command has been processed into the log. The only option is to send the command again. This means that a command that is meant to be processes once could be processed more than once. Commands must be written to ensure that this does not result in an inconsistent dataset. The easiest way to ensure this is to make sure the client gives each command a unique ID, a UUID. The server will then ignore any subsequent commands with the same id, and send a confirm reply back anyway.
-
-The server must thus keep a log of all processed command id's. This can potentially grow so big as to impact performance, so there needs to be a timelimit on this processed log. We can enforce this by requiring a UUID with a timestamp, so either UUID V1 or V2. Any commands with a UUID older than the timelimit will be denied.

package/design/commands.md

@@ -1,25 +0,0 @@
-# commands (CQRS)
-
-Updates/changes can only be done through commands. It uses a seperate endpoint (POST /update). There is only one process that handles this endpoint.
-
-Each command must have a unique uuid, set by the client. A command is handled asynchronously, when fetching /update you will only get a response like 'command accepted'.
-
-When the command is executed, the client is notified of the result (success or failure). This can be through a server-sent event, or though (long) polling. There should be an extra endpoint for each command, say GET /update/{uuid}
-
-Commands are executed only once, but may be received more than once, this is why the client assigns the uuid. Commands with the same uuid (within a specified time window, say one day) are ignored.
-
-commands are written to the command log. When this log is synced, then a response 'command accepted' is sent.
-
-Each server can/should define its own commands, with as much semantics as possible. So instead of defining simple CRUD commands, a server should have meaningfull commands, with opaque inner workings.
-
-A simple, but wrong, solution would be to implement a generic patch command, which uses jsonpatch. This woiuld allow for atomicity, thus fullfilling ACID requirements, but you cannot deduce from the command what the meaning of the change is. And you cannot change the data structure and command handling and then re-run all commands.
-
-However if you create a data structure for support tickets, you could create a command 'createTicket', which would know what to change and where in the dataset. If the dataset changes, you can change the code in the createTicket command handler in tandem. This avoids most of the problems related to schema changes. And this means there is less need of versioning of the API.
-
-You may still need to have versions of commands, so that if parameters to a command change, you can sense old commands through a version number.
-
-Similarly, the query endpoint will need versioning to handle changes in the dataset structure, as all code is defined on the client side. This may be mitigated somewhat by providing a custom library of semantic methods for the client to use, e.g. `searchTickets(...)`.
-
-Older versions of the query api can be simulated through a transformer, which translates/transforms from/to the new structure.
-
-Commands can change the datastructure, by creating a new immutable root and then starving/stopping the current VM processes and starting new VM processes with the new root.

package/design/identity.md

@@ -1,71 +0,0 @@
-# Identity
-
-One of many pitfalls in REST API's is mismanaging of the identity of entities. So we need to get this right here.
-
-- Each entity should have exactly one identity, over time. You can have different versions of an entity, as in older versions that have been updated since. The main identity always points to the latest version in that case.
-- Each identity should be a URL. This is because we want to make sure we can easily upgrade the API to a Linked Data API.
-- If you implement versioning, you should be able to reference a specific version, also through a URL.
-- If the JSONTag data adds a link to an object/entity, it should use the same ID (URL)
-- Each entity must have a unique ID
-
-A simple solution would be to use the API URL + JSON Pointer as the id of all entities. This fails for these reasons (at least):
-- Entities may appear on multiple JSON Pointer paths, as they can be linked in multiple places
-- Entities that are part of an array of entities, may have there JSON Pointer path changed when another, prior, entity is removed from the array. e.g.
-
-```jsontag
-[
-	{
-		"title":"Entity 1"
-	},
-	{
-		"title":"Entity 2"
-	}
-]
-```
-
-Here you could select entity 2 using the JSON Pointer '/1/'. However as soon as entity 1 is removed, the JSON Pointer would become '/0/'.
-
-Another solution would be to explicitly assign all objects a unique id, perhaps a UUID. Then you can use that with a specific Map container, e.g.
-
-```
-/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e
-```
-
-And all results would have their id encoded like this:
-
-```jsontag
-[
-	<object id="/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e">{
-		"title":"Entity 2"
-	}
-]
-```
-
-The problem here is that the system now forces everything to use a specific UUID type as ID. This may not be the best option. And the URL that is used as the ID is much less expressive than it could be. All entities are forced to use the same non-descriptive ID style. The benefit is that at least some common pitfalls in assigning ID values are avoided...
-
-The /uuid/ map should be invisible, as in, it should not be returned in any result. You can only use it to query a specific entity by ID.
-UUID's should only be assigned to object values. All other values are supposed to be just values, even if they get parsed into Value Objects, like `<date>`.
-
-One possible problem is that we've now encoded the identity as an attribute, instead of a property. This may come as a surprise to users of the API. However I do think that it is a better fit. Consider the JSON-LD '@id' property, which fulfills a similar role. It too is encoded seperate from normal properties, using the '@' prefix.
-
-The /uuid/ endpoint can be written as a search in the dataset for an entity with the given ID. This way it is not part of the dataset itself, and thus 'invisible'.
-
-If you need versioning, you could add a second parameter for the version, like this:
-
-```
-/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e/1
-```
-
-or
-
-```
-/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e/latest
-```
-
-To prevent collisions with a 'uuid' property in the dataset root, a normal JSON Pointer for the dataset could start with the `/query/` root path, e.g.:
-
-```
-api.url/query/tasks/
-```
-
-This would also make room for a seperate `/update/` path, which would handle update commands to the dataset.

package/design/immutability.md

@@ -1,26 +0,0 @@
-# Immutable JSONTag data
-
-1 - When parsing jsontag data, each entry is frozen immediately after instantiation.
-2 - Add a clone method to clone an object (shallow clone)
-3 - a clone is mutable
-4 - all objects linking to the original object get cloned as well, and the link updated to the new clone
-5 - once all updates are done, you can freeze the root object again and it will also freeze all clones
-6 - identity. Each object should have a clear identity, that is the same over mutations, as well as a static identity that changes with mutations. (one indentity-over-time, one identity-per-version)
-7 - cleanup. Any object no longer linked should be removed, unless you want automatic versioning. In that case the root objects versions should all be kept, and therefor all objects remain linked.
-
-## Problems
-
-### 4 - all objects linking to the original object get cloned as well
-
-This means that we need an index for each object, containing all other objects linking to it.
-We can do this while parsing the JSONTag data, just make a weakmap for all objects, while treewalking the dataset.
-
-### which acl grants apply to objects that are linked more than once?
-
-The simplest solution seems to just use the acl metadata gathered while following the path used.
-
-## existing solutions
-
-- immutable.js
-- immer
-- redux