@muze-nl/simplystore 0.1.2

package/.dockerignore

@@ -0,0 +1,8 @@
+.git/
+design/
+node_modules/
+
+.dockerignore
+Dockerfile
+LICENSE
+README.md

package/.github/workflows/docker-build.yml

@@ -0,0 +1,45 @@
+---
+name: Build Docker Image
+
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+    branches: [ main, master ]
+  # Allow manually triggering the workflow.
+  workflow_dispatch:
+
+# Cancels all previous workflow runs for the same branch that have not yet completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create docker tag (from git reference)
+        # A tag name may only contain lower- and uppercase letters, digits, underscores, periods and dashes.
+        run: |
+          echo "TAG=$(echo -n "${{ github.ref_name }}" \
+            | tr --complement --squeeze-repeats '[:alnum:]._-' '_')" \
+            >> "${GITHUB_ENV}"
+
+      - uses: actions/checkout@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Docker Image
+        run: |
+          docker build \
+            --tag "ghcr.io/poef/jsontag-rest-server:${{ env.TAG }}" \
+          .
+          docker push "ghcr.io/poef/jsontag-rest-server:${{ env.TAG }}"

package/.gitignore~

@@ -0,0 +1 @@
+node_modules

package/Dockerfile

@@ -0,0 +1,60 @@
+# ==============================================================================
+# Dependencies
+# ------------------------------------------------------------------------------
+FROM node:18-alpine3.17 as builder
+
+WORKDIR /usr/src/app
+COPY package.json ./
+
+# @TODO: Once development is stable, `npm ci --only=production` should be used instead of `npm install`
+RUN npm install --omit=dev
+# ==============================================================================
+
+
+# ==============================================================================
+# Application
+# ------------------------------------------------------------------------------
+FROM alpine:3.17
+
+ENV NODE_ENV=production
+
+RUN addgroup -g 1000 node \
+    && adduser -u 1000 -G node -s /bin/sh -D node \
+    && apk add --no-cache nodejs=~18.14
+
+COPY --chown=node:node . /app
+COPY --chown=node:node --from=builder /usr/src/app/node_modules /app/node_modules
+
+WORKDIR /app
+
+CMD [ "node", "/app/src/main.mjs" ]
+# ==============================================================================
+
+
+# ==============================================================================
+# Metadata
+# ------------------------------------------------------------------------------
+# @TODO: Once development is stable, the following should also be added
+#    org.label-schema.build-date=${BUILD_DATE} \ # Usually $(date --iso-8601=seconds)
+#    org.label-schema.vcs-ref=${BUILD_REF} \ # Usually $(git describe --tags --always)
+#    org.label-schema.version=${VERSION} \ # Usually $(git describe --tags --abbrev=0)
+#    org.opencontainers.image.created="${BUILD_DATE}" \
+#    org.opencontainers.image.version="${VERSION}"
+LABEL maintainer="Auke van Slooten <auke@muze.nl>" \
+    org.label-schema.description="JSONTag REST Server" \
+    org.label-schema.docker.cmd='docker run --expose 3000 --interactive --name=jsontag-rest-server --rm --tty --volume "\$PWD/my-data.json:/app/data.jsontag" jsontag-rest-server' \
+    org.label-schema.schema-version="1.0" \
+    org.label-schema.name="jsontag-rest-server" \
+    org.label-schema.url="https://github.com/poef/jsontag-rest-server" \
+    org.label-schema.usage="https://github.com/poef/jsontag-rest-server" \
+    org.label-schema.vcs-url="https://github.com/poef/jsontag-rest-server" \
+    org.label-schema.vendor="Auke van Slooten" \
+    org.opencontainers.image.authors="Auke van Slooten <auke@muze.nl>" \
+    org.opencontainers.image.description="JSONTag REST Server" \
+    org.opencontainers.image.documentation="https://github.com/poef/jsontag-rest-server" \
+    org.opencontainers.image.licenses="MIT" \
+    org.opencontainers.image.source="https://github.com/poef/jsontag-rest-server" \
+    org.opencontainers.image.title="jsontag-rest-server" \
+    org.opencontainers.image.url="https://github.com/poef/jsontag-rest-server" \
+    org.opencontainers.image.vendor="Auke van Slooten"
+# ==============================================================================

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Muze
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,99 @@
+# SimplyStore
+
+SimplyStore is an attempt to create a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. All data is read into memory and is available to these javascript queries without needing (or allowing) disk access or indexes.
+
+[JSONTag](https://github.com/poef/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags.
+Javascript queries are run in a [VM2](https://www.npmjs.com/package/vm2) sandbox.
+
+
+## Installation
+
+SimplyStore is a Node application. Start it by downloading this git repository:
+
+```shell
+git clone git@github.com:simplyedit/simplystore
+```
+
+Then install its dependencies:
+
+```shell
+cd simplystore
+npm install
+```
+
+And start it up:
+
+```shell
+npm start 
+```
+
+The server comes with a small demo dataset, which you can take a look at here `http://localhost:3000/`:
+
+This page will show this:
+
+```
+{
+    "persons":<link>"persons/"
+}
+```
+
+By following the link to `http://localhost:3000/persons/` you get:
+
+```
+[
+    <object class="Person">{
+        "name":"John",
+        "dob":<date>"1972-09-20"
+    },
+    <object class="Person">{
+        "name":"Jane",
+        "dob":<date>"1986-01-01"
+    }
+]
+```
+
+## Goals of this project
+
+SimplyStore is an attemp to see if we can create a more defined and usable REST like service, out of the box. One where all you need to do is change the data and add some access rights and get a self-describing, browseable, working API.
+
+The SimplyStore design is predicated on the following realisations:
+
+  1. Most data today will fit comfortably in memory in a commodity server.
+  2. REST today is usually JSON-over-HTTP, but JSON crucially misses a <link> type.
+  3. JSON is never just JSON. You need additional things like JSON-LD or JSON-Schema, to make sense of it. 
+  4. There is no clear onramp from JSON to Linked Data.
+  5. Linked Data is very good for data / information exchange, but very costly for data manipulation and querying.
+
+So the scope for jsontag-rest-server is:
+
+- datasets that will fit comfortably in memory, for now I've set a test goal of about 1GB of data.
+- usecases that are mostly-read, with sparse updates.
+- scale-in-depth, so scale up is limited to the limits of a single computer system
+- linked data (RDF et al) is not an immediate concern, but there must be a plausible onramp / conversion to and from linked data.
+
+In addition, SimplyStore is meant to be a real-world testcase for JSONTag.
+
+## Roadmap
+
+- [v] immutable dataset
+- allow changes to dataset by creating a new root
+- command handling with crud commands and command log
+- backup current dataset to JSONTag file
+- on startup check if any commands in the log haven't been resolved, if so run them
+
+- improved web client with type-specific views and form elements
+
+- Datalog query support
+  - [v] compile triple store from jsontag data
+  - [v] add query method
+  - [v] extend datalog query to allow for custom match functions
+  - [v] run /query post body in VM2 sandbox
+  - [v] immutable dataset in query vm
+  - add indexing and other optimizations
+  - add standard library of matching functions
+  - allow vanilla javascript array map/reduce/filter approach
+
+- add support for metadata on each JSON pointer path (or better: each object)
+- allow custom templates, instead of the default index.html
+- add support for access control, based on webid / openid connect
+

package/data.jsontag

@@ -0,0 +1,20 @@
+{
+	"persons": [
+		<object id="john" class="Person">{
+			"name": "John",
+			"lastName": "Doe",
+			"dob": <date>"1972-09-20",
+			"foaf": [
+				<link>"jane"
+			]
+		},
+		<object id="jane" class="Person">{
+			"name": "Jane",
+			"lastName": "Doe",
+			"dob": <date>"1986-01-01",
+			"foaf": [
+				<link>"john"
+			]
+		}
+	]
+}

package/design/access-management.md

@@ -0,0 +1,25 @@
+# access management (grants/rights/roles)
+
+## Read grants
+
+You can easily create a read access check that just checks the query/ endpoint, but this is not interesting.
+Much more interesting is how to implement granular read access rights on subtrees of the dataset.
+
+Ariadne has a mechanism that has been succesfully used since 1998. It allows you to defined grant strings, like 'read', 'edit', etc. and configure them on a path inside a tree of data. Each entity below that path will automatically inherit the grant.
+
+The grant is then checked for each interaction with the data. Because all interactions in Ariadne are through templates, which can be custom made, the grants can also be custom strings. This way you can grow the access management system to your own needs.
+
+However, this means that all read access must come through the data tree. Earlier we defined a /uuid/ endpoint which would give direct access to any object. This breaks this paradigm. Objects can be linked in multiple locations in the dataset. To check for read grants, you must find all valid paths to an object and check if any of them allow the user to 'read' the object. This is potentially very costly.
+
+The easiest solution is to drop the /uuid endpoint, unless all data is publically readable. Each object can still have a uuid, but it is no longer a url that points to the objects contents.
+
+Another problem is the changing nature of the JSON Pointer path in the URL. If an object is part of an array, and another object earlier (with smalled index) is removed, the URL for this object changes. Its index is lowered. So you cannot assign grants on a JSON Pointer path, containing an array index, with any hope of the grants staying in the correct spot.
+
+If you instead assign grants to objects directly, the grants will correctly move when an array is updated. However this opens up the possibility that grants appear in multiple places, if the object is linked in multiple places. This should not be a problem though, since the subtree of objects is the same as long as the object is the same.
+
+This opens up the possibility of storing the grants in an attribute on the JSONTag tag of the object. e.g.
+
+<object class="Site" grants="user1: read edit, user2: read edit delete">{ ... }
+
+Here we need to carefully consider how to treat links. Do we need read access to follow a link, or is that implied?
+

package/design/acid.md

@@ -0,0 +1,31 @@
+# JSONTAG REST Server - Commands / CQRS
+
+Why the need for CQRS (Command Query Responsibility Segregation?)
+
+The design of the server is purposely simple. It does away with a seperate database system, instead reading all data in memory, giving access to that data to queries written in javascript, using native javascript objects. This does away with the relational-object impedance mismatch. It does away with SQL and ORM solutions.
+
+However, databases have the nice property that they are ACID compliant. We don't want to lose that. ACID stands for 'Atomicity', 'Consistency', 'Isolation' and 'Durability'. Our server should exhibit these same properties. Isolation is handled by handling each query in a seperate VM, using immutable data. But immutable data also means we need a different mechanism to update data, other than plain javascript.
+
+Why the need for immutable data? This is tied to the ACID requirements. If you have multiple processes working with the same, shared, data, there is a good chance of creating inconsistent data. One process changes the data in some way, while another process tries to change the exact same data. One of these processes will 'win', potentially undoing the other process' change. Imagine you have a shop with an inventory. If product X has 1 item in the inventory, and there are two requests to buy that product being handled by two different processes at the same time, you could end up selling the same item twice.
+
+There are ways around this, using locks and mutexes, but these are notoriously difficult to get right. 
+
+Just using immutable data alone won't solve this problem. Both processes will still see that there is 1 product X in store. But if we only allow updates / changes, through a single sequential process, and we allow for the possibility that a change request can fail, we can solve this problem much easier.
+
+So both processes see that there is one item left in inventory. Both processes now create a command (buyProduct(X)) and send it to the command queue. They both get a unique ID as a result, and they can listen for an update on that ID. But there is only a single command handler, which handles commands on a First In First Out basis. The first buyProduct(X) command succeeds, and the inventory is decreased by 1. The next buyProduct(X) command fails its precondition, the inventory is 0. So it fails. Both processes get an update, one is a success, the other is a failure.
+
+Now as the data is immutable, this is not entirely correct. In fact there is a difference between the client process calling the server, and the server process which only allows you to query the data. The server process can't create a command or listen for updates, only the client can. So when the client receives a success or failure resutl, asynchronously, it can send a new query to the server. The server in the mean time creates a new query handler, with the updated data after the changes from the last commands.
+
+Because changes are only applied using the single process command handler, Atomicity is preserved. Each command is an atomic update, it either succeeds of fails. If it fails, none of the updates are kept, since the immutable data structure root is not updated.
+
+Consistency is preserved. Each query request sees a concistent dataset, it may be slightly out of date, but it is internally consistent. Later queries will see a more up to date version of the data, so in the context of updates, the data is eventually consistent.
+
+Isolation is preserved. Each query runs in its own VM, with its own access to the immutable data. Each command is run seperate from queries and custom javascript. 
+
+Durability is preserved. Each command is first written to a log, then the unique ID of the command is returned. The whole dataset is backup to disk once in a while, with the last command ID that has been processed. A backup can be restored, then all commands after the last processes one can be processed again.
+
+The query handlers can run parallel, since they only have access to immutable data. In fact, each query handler process can use the exact same shared memory, keeping memory usage low.
+
+There is a potential problem that a client sends a command, which the server enters into the command log, but before the command log id can be sent back to the client, the server dies. In that case the client cannot know if the command has been processed into the log. The only option is to send the command again. This means that a command that is meant to be processes once could be processed more than once. Commands must be written to ensure that this does not result in an inconsistent dataset. The easiest way to ensure this is to make sure the client gives each command a unique ID, a UUID. The server will then ignore any subsequent commands with the same id, and send a confirm reply back anyway.
+
+The server must thus keep a log of all processed command id's. This can potentially grow so big as to impact performance, so there needs to be a timelimit on this processed log. We can enforce this by requiring a UUID with a timestamp, so either UUID V1 or V2. Any commands with a UUID older than the timelimit will be denied.

package/design/commands.md

@@ -0,0 +1,25 @@
+# commands (CQRS)
+
+Updates/changes can only be done through commands. It uses a seperate endpoint (POST /update). There is only one process that handles this endpoint.
+
+Each command must have a unique uuid, set by the client. A command is handled asynchronously, when fetching /update you will only get a response like 'command accepted'.
+
+When the command is executed, the client is notified of the result (success or failure). This can be through a server-sent event, or though (long) polling. There should be an extra endpoint for each command, say GET /update/{uuid}
+
+Commands are executed only once, but may be received more than once, this is why the client assigns the uuid. Commands with the same uuid (within a specified time window, say one day) are ignored.
+
+commands are written to the command log. When this log is synced, then a response 'command accepted' is sent.
+
+Each server can/should define its own commands, with as much semantics as possible. So instead of defining simple CRUD commands, a server should have meaningfull commands, with opaque inner workings.
+
+A simple, but wrong, solution would be to implement a generic patch command, which uses jsonpatch. This woiuld allow for atomicity, thus fullfilling ACID requirements, but you cannot deduce from the command what the meaning of the change is. And you cannot change the data structure and command handling and then re-run all commands.
+
+However if you create a data structure for support tickets, you could create a command 'createTicket', which would know what to change and where in the dataset. If the dataset changes, you can change the code in the createTicket command handler in tandem. This avoids most of the problems related to schema changes. And this means there is less need of versioning of the API.
+
+You may still need to have versions of commands, so that if parameters to a command change, you can sense old commands through a version number.
+
+Similarly, the query endpoint will need versioning to handle changes in the dataset structure, as all code is defined on the client side. This may be mitigated somewhat by providing a custom library of semantic methods for the client to use, e.g. `searchTickets(...)`.
+
+Older versions of the query api can be simulated through a transformer, which translates/transforms from/to the new structure.
+
+Commands can change the datastructure, by creating a new immutable root and then starving/stopping the current VM processes and starting new VM processes with the new root.

package/design/identity.md

@@ -0,0 +1,71 @@
+# Identity
+
+One of many pitfalls in REST API's is mismanaging of the identity of entities. So we need to get this right here.
+
+- Each entity should have exactly one identity, over time. You can have different versions of an entity, as in older versions that have been updated since. The main identity always points to the latest version in that case.
+- Each identity should be a URL. This is because we want to make sure we can easily upgrade the API to a Linked Data API.
+- If you implement versioning, you should be able to reference a specific version, also through a URL.
+- If the JSONTag data adds a link to an object/entity, it should use the same ID (URL)
+- Each entity must have a unique ID
+
+A simple solution would be to use the API URL + JSON Pointer as the id of all entities. This fails for these reasons (at least):
+- Entities may appear on multiple JSON Pointer paths, as they can be linked in multiple places
+- Entities that are part of an array of entities, may have there JSON Pointer path changed when another, prior, entity is removed from the array. e.g.
+
+```jsontag
+[
+	{
+		"title":"Entity 1"
+	},
+	{
+		"title":"Entity 2"
+	}
+]
+```
+
+Here you could select entity 2 using the JSON Pointer '/1/'. However as soon as entity 1 is removed, the JSON Pointer would become '/0/'.
+
+Another solution would be to explicitly assign all objects a unique id, perhaps a UUID. Then you can use that with a specific Map container, e.g.
+
+```
+/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e
+```
+
+And all results would have their id encoded like this:
+
+```jsontag
+[
+	<object id="/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e">{
+		"title":"Entity 2"
+	}
+]
+```
+
+The problem here is that the system now forces everything to use a specific UUID type as ID. This may not be the best option. And the URL that is used as the ID is much less expressive than it could be. All entities are forced to use the same non-descriptive ID style. The benefit is that at least some common pitfalls in assigning ID values are avoided...
+
+The /uuid/ map should be invisible, as in, it should not be returned in any result. You can only use it to query a specific entity by ID.
+UUID's should only be assigned to object values. All other values are supposed to be just values, even if they get parsed into Value Objects, like `<date>`.
+
+One possible problem is that we've now encoded the identity as an attribute, instead of a property. This may come as a surprise to users of the API. However I do think that it is a better fit. Consider the JSON-LD '@id' property, which fulfills a similar role. It too is encoded seperate from normal properties, using the '@' prefix.
+
+The /uuid/ endpoint can be written as a search in the dataset for an entity with the given ID. This way it is not part of the dataset itself, and thus 'invisible'.
+
+If you need versioning, you could add a second parameter for the version, like this:
+
+```
+/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e/1
+```
+
+or
+
+```
+/uuid/9b91dcfd-dae7-47dc-b90f-a51b37e6dd3e/latest
+```
+
+To prevent collisions with a 'uuid' property in the dataset root, a normal JSON Pointer for the dataset could start with the `/query/` root path, e.g.:
+
+```
+api.url/query/tasks/
+```
+
+This would also make room for a seperate `/update/` path, which would handle update commands to the dataset.

package/design/immutability.md

@@ -0,0 +1,26 @@
+# Immutable JSONTag data
+
+1 - When parsing jsontag data, each entry is frozen immediately after instantiation.
+2 - Add a clone method to clone an object (shallow clone)
+3 - a clone is mutable
+4 - all objects linking to the original object get cloned as well, and the link updated to the new clone
+5 - once all updates are done, you can freeze the root object again and it will also freeze all clones
+6 - identity. Each object should have a clear identity, that is the same over mutations, as well as a static identity that changes with mutations. (one indentity-over-time, one identity-per-version)
+7 - cleanup. Any object no longer linked should be removed, unless you want automatic versioning. In that case the root objects versions should all be kept, and therefor all objects remain linked.
+
+## Problems
+
+### 4 - all objects linking to the original object get cloned as well
+
+This means that we need an index for each object, containing all other objects linking to it.
+We can do this while parsing the JSONTag data, just make a weakmap for all objects, while treewalking the dataset.
+
+### which acl grants apply to objects that are linked more than once?
+
+The simplest solution seems to just use the acl metadata gathered while following the path used.
+
+## existing solutions
+
+- immutable.js
+- immer
+- redux