antietcd 1.1.3 → 1.2.0

package/README.md

@@ -5,6 +5,8 @@ Simplistic miniature etcd replacement based on [TinyRaft](https://git.yourcmc.ru
 - Embeddable
 - REST API only, gRPC is shit and will never be supported
 - [TinyRaft](https://git.yourcmc.ru/vitalif/tinyraft/)-based leader election
+- [Strict serializable](https://jepsen.io/consistency/models/strict-serializable) transaction isolation,
+  confirmed by [Jepsen tests](https://git.yourcmc.ru/vitalif/antietcd/src/branch/master/jepsen)
 - Websocket-based cluster communication
 - Supports a limited subset of etcd REST APIs
 - With optional persistence
@@ -52,6 +54,7 @@ node_modules/.bin/anticli [OPTIONS] put <key> [<value>]
 node_modules/.bin/anticli [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only] [--no-temp]
 node_modules/.bin/anticli [OPTIONS] del <key> [-p|--prefix]
 node_modules/.bin/anticli [OPTIONS] load [--with-lease] < dump.json
+node_modules/.bin/anticli [OPTIONS] watch <key> [key...] [-p|--prefix]
 ```
 
 For `put`, if `<value>` is not specified, it will be read from STDIN.
@@ -69,9 +72,15 @@ Options:
 <dt>--key &lt;key&gt;</dt>
 <dd>Use TLS with this key file (PEM format)</dd>
 
+<dt>--ca &lt;cert&gt;</dt>
+<dd>Use this TLS CA certificate to verify server certificate</dd>
+
 <dt>--timeout 1000</dt>
 <dd>Specify request timeout in milliseconds</dd>
 
+<dt>--base64 1</dt>
+<dd>Use base64 encoding of keys and values, like in etcd (enabled by default)</dd>
+
 <dt>--json or --write-out=json</dt>
 <dd>Print raw response in JSON</dd>
 
@@ -110,6 +119,22 @@ Specify &lt;ca&gt; = &lt;cert&gt; if your certificate is self-signed.</dd>
 
 </dl>
 
+### Consistency
+
+<dl>
+
+<dt>--use_locks 1</dt>
+<dd>Enable lock-based transaction isolation to prevent <a href="https://jepsen.io/consistency/phenomena/g1a">G1a</a>.</dd>
+
+<dt>--merge_watches 1</dt>
+<dd>Antietcd merges all watcher events for a single transaction into a single websocket message to provide
+more ordering/transaction guarantees. Set to 0 to disable this behaviour.</dd>
+
+<dt>--stale_read 1</dt>
+<dd>Allow to serve reads from replicas (potentially stale). Specify 0 to always contact leader on every read.</dd>
+
+</dl>
+
 ### Persistence
 
 <dl>
@@ -152,12 +177,12 @@ for every change and returning a new value or undefined to skip persistence.</dd
 <dt>--wait_quorum_timeout 30000</dt>
 <dd>Timeout for requests to wait for quorum to come up</dd>
 
+<dt>--leadership_timeout 3000</dt>
+<dd>TinyRaft leadership timeout</dd>
+
 <dt>--leader_priority &lt;number&gt;</dt>
 <dd>Raft leader priority for this node (optional)</dd>
 
-<dt>--stale_read 1</dt>
-<dd>Allow to serve reads from followers. Specify 0 to disallow</dd>
-
 <dt>--reconnect_interval 1000</dt>
 <dd>Unavailable peer connection retry interval</dd>
 
@@ -178,6 +203,21 @@ for every change and returning a new value or undefined to skip persistence.</dd
 
 </dl>
 
+### Logging
+
+<dl>
+
+<dt>--logs keyword1,keyword2,...</dt>
+<dd>
+Enable logs by keywords:
+
+- access: log all requests and incoming websocket messages.
+- watch: log outgoing websocket watch messages.
+-cluster: log clustering (TinyRaft) events.
+</dd>
+
+</dl>
+
 ## Embedded Usage
 
 ```js
@@ -264,6 +304,15 @@ function example_filter(cfg)
 module.exports = example_filter;
 ```
 
+## Consistency Guarantees
+
+When `--stale_read` is enabled, Antietcd provides [SERIALIZABLE](https://jepsen.io/consistency/models/serializable) transaction isolation level.
+
+When it's disabled, the level is [STRICT SERIALIZABLE](https://jepsen.io/consistency/models/strict-serializable).
+
+Reconnected watchers don't receive all historical events individually, but they are guaranteed to receive
+a consistent snapshot of keys in interest.
+
 ## Supported etcd APIs
 
 NOTE: `key`, `value` and `range_end` are always encoded in base64, like in original etcd.
@@ -527,6 +576,7 @@ type ServerMessage = {
 - 400 for invalid requests
 - 404 for unsupported API / URL not found
 - 405 for non-POST request method
+- 408 for lock wait timeouts
 - 501 for unsupported API feature - non-directory range queries and so on
 - 502 for server is stopping
 - 503 for quorum-related errors - quorum not available and so on

package/anticli.js

@@ -8,6 +8,7 @@ const fs = require('fs');
 const fsp = require('fs').promises;
 const http = require('http');
 const https = require('https');
+const WebSocket = require('ws');
 
 const help_text = `CLI for AntiEtcd
 (c) Vitaliy Filippov, 2024
@@ -18,12 +19,13 @@ Usage:
 anticli.js [OPTIONS] put <key> [<value>]
 anticli.js [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only] [--no-temp]
 anticli.js [OPTIONS] del <key> [-p|--prefix]
+anticli.js [OPTIONS] watch <key> [key...] [-p|--prefix]
 anticli.js [OPTIONS] load [--with-lease] < dump.json
 
 Options:
 
 [--endpoints|-e http://node1:2379,http://node2:2379,http://node3:2379]
-[--cert cert.pem] [--key key.pem] [--timeout 1000] [--json]
+[--cert cert.pem] [--key key.pem] [--ca ca.pem] [--timeout 1000] [--base64 1] [--json]
 `;
 
 class AntiEtcdCli
@@ -83,9 +85,9 @@ class AntiEtcdCli
                 cmd.push(arg);
             }
         }
-        if (!cmd.length || cmd[0] != 'get' && cmd[0] != 'put' && cmd[0] != 'del' && cmd[0] != 'load')
+        if (!cmd.length || cmd[0] != 'get' && cmd[0] != 'put' && cmd[0] != 'del' && cmd[0] != 'load' && cmd[0] != 'watch')
         {
-            process.stderr.write('Supported commands: get, put, del, load. Use --help to see details\n');
+            process.stderr.write('Supported commands: get, put, del, load, watch. Use --help to see details\n');
             process.exit(1);
         }
         return [ cmd, options ];
@@ -98,12 +100,16 @@ class AntiEtcdCli
         {
             this.options.endpoints = [ 'http://localhost:2379' ];
         }
+        this.options.base64 = (this.options.base64 == null) || is_true(this.options.base64);
+        this.tls = {};
         if (this.options.cert && this.options.key)
         {
-            this.tls = {
-                key: await fsp.readFile(this.options.key),
-                cert: await fsp.readFile(this.options.cert),
-            };
+            this.tls.key = await fsp.readFile(this.options.key);
+            this.tls.cert = await fsp.readFile(this.options.cert);
+        }
+        if (this.options.ca)
+        {
+            this.tls.ca = await fsp.readFile(this.options.ca);
         }
         if (cmd[0] == 'get')
         {
@@ -121,6 +127,10 @@ class AntiEtcdCli
         {
             await this.load();
         }
+        else if (cmd[0] == 'watch')
+        {
+            await this.watch(cmd.slice(1));
+        }
         // wait until output is fully flushed
         await new Promise(ok => process.stdout.write('', ok));
         await new Promise(ok => process.stderr.write('', ok));
@@ -168,7 +178,9 @@ class AntiEtcdCli
         {
             keys = keys.map(k => k.replace(/\/+$/, ''));
         }
-        const txn = { success: keys.map(key => ({ request_range: this.options.prefix ? { key: b64(key+'/'), range_end: b64(key+'0') } : { key: b64(key) } })) };
+        const txn = { success: keys.map(key => ({ request_range: this.options.prefix
+            ? { key: this.b64(key+'/'), range_end: this.b64(key+'0') }
+            : { key: this.b64(key) } })) };
         const res = await this.request('/v3/kv/txn', txn);
         if (this.options.notemp)
         {
@@ -194,11 +206,11 @@ class AntiEtcdCli
                 {
                     if (!this.options.print_value_only)
                     {
-                        process.stdout.write(de64(kv.key)+'\n');
+                        process.stdout.write(this.de64(kv.key)+'\n');
                     }
                     if (!this.options.keys_only)
                     {
-                        process.stdout.write(de64(kv.value)+'\n');
+                        process.stdout.write(this.de64(kv.value)+'\n');
                     }
                 }
             }
@@ -211,7 +223,7 @@ class AntiEtcdCli
         {
             value = await new Promise((ok, no) => fs.readFile(0, { encoding: 'utf-8' }, (err, res) => err ? no(err) : ok(res)));
         }
-        const res = await this.request('/v3/kv/put', { key: b64(key), value: b64(value) });
+        const res = await this.request('/v3/kv/put', { key: this.b64(key), value: this.b64(value) });
         if (res.header)
         {
             process.stdout.write('OK\n');
@@ -224,7 +236,9 @@ class AntiEtcdCli
         {
             keys = keys.map(k => k.replace(/\/+$/, ''));
         }
-        const txn = { success: keys.map(key => ({ request_delete_range: this.options.prefix ? { key: b64(key+'/'), range_end: b64(key+'0') } : { key: b64(key) } })) };
+        const txn = { success: keys.map(key => ({ request_delete_range: this.options.prefix
+            ? { key: this.b64(key+'/'), range_end: this.b64(key+'0') }
+            : { key: this.b64(key) } })) };
         const res = await this.request('/v3/kv/txn', txn);
         for (const r of res.responses||[])
         {
@@ -235,6 +249,75 @@ class AntiEtcdCli
         }
     }
 
+    async watch(keys)
+    {
+        const prefix = this.options.prefix || !keys.length;
+        keys = keys.length ? keys : [''];
+        for (const url of this.options.endpoints)
+        {
+            const ws = new WebSocket(url.replace(/^http/, 'ws').replace(/\/+$/, '')+'/watch', this.tls||{});
+            let closed = false;
+            let started = false;
+            let close_cb = null;
+            const on_error = (e) =>
+            {
+                console.log(e);
+                closed = true;
+                if (close_cb)
+                    close_cb();
+            };
+            ws.on('error', on_error);
+            ws.on('message', (msg) =>
+            {
+                try
+                {
+                    msg = JSON.parse(msg);
+                }
+                catch (e)
+                {
+                    console.error(JSON.stringify('Bad JSON received from websocket: '+msg));
+                    return;
+                }
+                if (this.options.base64 && msg.result && msg.result.events)
+                {
+                    for (const ev of msg.result.events)
+                    {
+                        if (ev.kv)
+                        {
+                            ev.kv.key = this.de64(ev.kv.key);
+                            ev.kv.value = this.de64(ev.kv.value);
+                        }
+                    }
+                }
+                console.log(JSON.stringify(msg));
+            });
+            ws.on('open', () =>
+            {
+                started = true;
+                for (let i = 0; i < keys.length; i++)
+                {
+                    const k = keys[i];
+                    ws.send(JSON.stringify({
+                        create_request: {
+                            key: this.b64(prefix ? k+'/' : k),
+                            range_end: prefix ? k+'0' : undefined,
+                            watch_id: i+1,
+                            progress_notify: true,
+                        },
+                    }));
+                }
+            });
+            if (closed)
+                continue;
+            else
+            {
+                await new Promise(ok => close_cb = ok);
+                if (!started)
+                    return;
+            }
+        }
+    }
+
     async request(path, body)
     {
         for (const url of this.options.endpoints)
@@ -267,6 +350,16 @@ class AntiEtcdCli
         }
         process.exit(1);
     }
+
+    b64(str)
+    {
+        return this.options.base64 ? Buffer.from(str).toString('base64') : str;
+    }
+
+    de64(str)
+    {
+        return this.options.base64 ? Buffer.from(str, 'base64').toString() : str;
+    }
 }
 
 function POST(url, options, body, timeout)
@@ -281,7 +374,7 @@ function POST(url, options, body, timeout)
             req = null;
             ok({ error: 'timeout' });
         }, timeout) : null;
-        let req = (url.substr(0, 6).toLowerCase() == 'https://' ? https : http).request(url, { method: 'POST', headers: {
+        let req = (url.substr(0, 8).toLowerCase() == 'https://' ? https : http).request(url, { method: 'POST', headers: {
             'Content-Type': 'application/json',
             'Content-Length': body_text.length,
         }, timeout, ...options }, (res) =>
@@ -320,14 +413,9 @@ function POST(url, options, body, timeout)
     });
 }
 
-function b64(str)
-{
-    return Buffer.from(str).toString('base64');
-}
-
-function de64(str)
+function is_true(s)
 {
-    return Buffer.from(str, 'base64').toString();
+    return s === true || s === 1 || s === '1' || s === 'yes' || s === 'true' || s === 'on';
 }
 
 new AntiEtcdCli().run(...AntiEtcdCli.parse(process.argv)).catch(console.error);

package/anticluster.js

@@ -9,10 +9,6 @@ const { runCallbacks, RequestError } = require('./common.js');
 
 const LEADER_MISMATCH = 'raft leader/term mismatch';
 
-const LEADER_ONLY = 1;
-const NO_WAIT_QUORUM = 2;
-const READ_FROM_FOLLOWER = 4;
-
 class AntiCluster
 {
     constructor(antietcd)
@@ -38,6 +34,7 @@ class AntiCluster
             nodeId: this.cfg.node_id,
             heartbeatTimeout: this.cfg.heartbeat_timeout,
             electionTimeout: this.cfg.election_timeout,
+            leadershipTimeout: this.cfg.leadership_timeout||(((this.cfg.heartbeat_timeout||1000) + (this.cfg.election_timeout||5000))/2),
             leaderPriority: this.cfg.leader_priority||undefined,
             initialTerm: this.antietcd.stored_term,
             send: (to, msg) => this._sendRaftMessage(to, msg),
@@ -86,7 +83,7 @@ class AntiCluster
 
     async replicateChange(msg)
     {
-        if (this.raft.state !== TinyRaft.LEADER)
+        if (this.raft.state !== TinyRaft.LEADER || !this.synced)
         {
             return;
         }
@@ -104,9 +101,14 @@ class AntiCluster
         }
     }
 
+    unlockChange(revision)
+    {
+        return this._requestFollowers({ unlock: { revision } }, this.cfg.replication_timeout||1000);
+    }
+
     _log(msg)
     {
-        if (this.cfg.log_level > 0)
+        if (this.cfg.logs.cluster)
         {
             console.log(msg);
         }
@@ -186,7 +188,6 @@ class AntiCluster
         {
             // (Re)sync with the new set of followers
             this._resync(event.followers);
-            this.antietcd.etctree.resume_leases();
         }
         else
         {
@@ -233,7 +234,7 @@ class AntiCluster
                                 );
                             }
                             this.resync_state.dumps[client.raft_node_id] = res.error ? null : res;
-                            this._continueResync();
+                            return this._continueResync();
                         }
                     });
                 }
@@ -246,10 +247,10 @@ class AntiCluster
                 delete this.resync_state.dumps[f];
             }
         }
-        this._continueResync();
+        this._continueResync().catch(console.error);
     }
 
-    _continueResync()
+    async _continueResync()
     {
         if (!this.resync_state ||
             Object.values(this.resync_state.dumps).filter(d => !d).length > 0)
@@ -286,31 +287,37 @@ class AntiCluster
             this._log(update_only ? 'Updating database from node '+with_max[i]+' state' : 'Copying node '+with_max[i]+' state');
             this.antietcd.etctree.load(this.resync_state.dumps[with_max[i]], update_only);
         }
+        if (this.antietcd.persistence)
+        {
+            await this.antietcd.persistence.persist();
+        }
+        this.antietcd.stored_term = this.raft.term;
+        if (this.cfg.use_locks)
+        {
+            this.antietcd.etctree.resync_notifications();
+            this.antietcd.locker.break_locks();
+        }
         let wait = 0;
         const load_request = { term: this.raft.term, load: this.antietcd.etctree.dump() };
         for (const follower in this.resync_state.dumps)
         {
             if (follower != this.cfg.node_id)
             {
-                const dump = this.resync_state.dumps[follower];
-                if (dump.term <= max_term)
+                const client = this._getPeer(follower);
+                if (!client)
                 {
-                    const client = this._getPeer(follower);
-                    if (!client)
-                    {
-                        this._log('Lost peer connection during resync - restarting election');
-                        this.raft.start();
-                        return;
-                    }
-                    this._log('Copying state to '+follower);
-                    const loadstate = this.resync_state.loads[follower] = {};
-                    wait++;
-                    this._peerRequest(client, load_request, this.cfg.load_timeout||5000).then(res =>
-                    {
-                        loadstate.result = res;
-                        this._finishResync();
-                    });
+                    this._log('Lost peer connection during resync - restarting election');
+                    this.raft.start();
+                    return;
                 }
+                this._log('Copying state to '+follower);
+                const loadstate = this.resync_state.loads[follower] = {};
+                wait++;
+                this._peerRequest(client, load_request, this.cfg.load_timeout||5000).then(res =>
+                {
+                    loadstate.result = res;
+                    this._finishResync();
+                });
             }
         }
         if (!wait)
@@ -328,7 +335,7 @@ class AntiCluster
             return;
         }
         // All current peers have copied the database, we can proceed
-        this.antietcd.stored_term = this.raft.term;
+        this.antietcd.etctree.resume_leases();
         this.synced = true;
         runCallbacks(this, 'wait_sync', []);
         this._log(
@@ -348,38 +355,48 @@ class AntiCluster
         return path != 'kv_range';
     }
 
-    async checkRaftState(path, leaderonly, data)
+    async checkRaftState(path, data)
     {
         if (!this.raft)
         {
             return null;
         }
-        if (leaderonly == LEADER_ONLY && this.raft.state != TinyRaft.LEADER)
+        if (data.leaderonly && this.raft.state != TinyRaft.LEADER)
         {
             throw new RequestError(503, 'Not leader');
         }
-        if (leaderonly == NO_WAIT_QUORUM && this.raft.state == TinyRaft.CANDIDATE)
-        {
-            throw new RequestError(503, 'Quorum not available');
-        }
-        if (!this.synced)
+        if (!data.nowaitquorum)
         {
-            // Wait for quorum / initial sync with timeout
-            await new Promise((ok, no) =>
+            if (this.raft.state == TinyRaft.CANDIDATE)
             {
-                this.wait_sync.push(ok);
-                setTimeout(() =>
+                throw new RequestError(503, 'Quorum not available');
+            }
+            if (!this.synced)
+            {
+                // Wait for quorum / initial sync with timeout
+                await new Promise((ok, no) =>
                 {
-                    this.wait_sync = this.wait_sync.filter(cb => cb != ok);
-                    no(new RequestError(503, 'Quorum not available'));
-                }, this.cfg.wait_quorum_timeout||30000);
-            });
+                    this.wait_sync.push(ok);
+                    setTimeout(() =>
+                    {
+                        this.wait_sync = this.wait_sync.filter(cb => cb != ok);
+                        no(new RequestError(503, 'Quorum not available'));
+                    }, this.cfg.wait_quorum_timeout||30000);
+                });
+            }
         }
-        if (this.raft.state == TinyRaft.FOLLOWER &&
-            (this._isWrite(path, data) || !this.cfg.stale_read && !(leaderonly & READ_FROM_FOLLOWER)))
+        if (this.raft.state == TinyRaft.FOLLOWER)
         {
-            // Forward to leader
-            return await this._forwardToLeader(path, data);
+            if (this._isWrite(path, data))
+            {
+                // Forward to leader
+                return await this._forwardToLeader(path, data);
+            }
+            else if (!this.cfg.stale_read && !data.serializable)
+            {
+                // Just check that the leader is still here and replicating to us
+                return await this._checkLeader();
+            }
         }
         return null;
     }
@@ -391,7 +408,27 @@ class AntiCluster
         {
             throw new RequestError(503, 'Leader is unavailable');
         }
-        return await this._peerRequest(client, { handler, request: data }, this.cfg.forward_timeout||1000);
+        const res = await this._peerRequest(client, { handler, request: data }, this.cfg.forward_timeout||1000);
+        if (res.error)
+        {
+            throw new RequestError(503, res.error);
+        }
+        return res;
+    }
+
+    async _checkLeader()
+    {
+        const client = this._getPeer(this.raft.leader);
+        if (!client)
+        {
+            throw new RequestError(503, 'Leader is unavailable');
+        }
+        const res = await this._peerRequest(client, { check_leader: true, term: this.raft.term }, this.cfg.forward_timeout||1000);
+        if (res.error)
+        {
+            throw new RequestError(503, res.error);
+        }
+        return res;
     }
 
     handleWsMsg(client, msg)
@@ -409,6 +446,10 @@ class AntiCluster
                 msg.identify.node_id != this.cfg.node_id)
             {
                 client.raft_node_id = msg.identify.node_id;
+                if (!this.cluster_connections[client.raft_node_id])
+                {
+                    this.cluster_connections[client.raft_node_id] = client.id;
+                }
                 this._log('Got a connection from '+client.raft_node_id);
             }
         }
@@ -432,6 +473,14 @@ class AntiCluster
         {
             this._handleCompactMsg(client, msg);
         }
+        else if (msg.unlock)
+        {
+            this._handleUnlockMsg(client, msg);
+        }
+        else if (msg.check_leader)
+        {
+            this._handleCheckLeaderMsg(client, msg);
+        }
     }
 
     async _handleRequestMsg(client, msg)
@@ -443,7 +492,10 @@ class AntiCluster
         }
         catch (e)
         {
-            console.error(e);
+            if (!(e instanceof RequestError))
+            {
+                console.error(e);
+            }
             client.socket.send(JSON.stringify({ request_id: msg.request_id, reply: { error: e.message } }));
         }
     }
@@ -460,6 +512,11 @@ class AntiCluster
             }
             this.antietcd.stored_term = msg.term;
             this.synced = true;
+            if (this.cfg.use_locks)
+            {
+                this.antietcd.etctree.resync_notifications();
+                this.antietcd.locker.break_locks();
+            }
             runCallbacks(this, 'wait_sync', []);
             this._log(
                 'Synchronized with leader, new term is '+this.raft.term+
@@ -502,6 +559,35 @@ class AntiCluster
         }
     }
 
+    _handleUnlockMsg(client, msg)
+    {
+        if (client.raft_node_id && this.raft.state == TinyRaft.FOLLOWER &&
+            this.raft.leader === client.raft_node_id && this.raft.term == msg.term)
+        {
+            if (this.cfg.use_locks)
+            {
+                this.antietcd.locker.unlock_txn(msg.unlock.revision);
+            }
+            client.socket.send(JSON.stringify({ request_id: msg.request_id, reply: {} }));
+        }
+        else
+        {
+            client.socket.send(JSON.stringify({ request_id: msg.request_id, reply: { error: LEADER_MISMATCH } }));
+        }
+    }
+
+    _handleCheckLeaderMsg(client, msg)
+    {
+        if (this.raft.state == TinyRaft.LEADER && this.raft.term == msg.term)
+        {
+            client.socket.send(JSON.stringify({ request_id: msg.request_id, reply: {} }));
+        }
+        else
+        {
+            client.socket.send(JSON.stringify({ request_id: msg.request_id, reply: { error: LEADER_MISMATCH } }));
+        }
+    }
+
     _getPeer(to)
     {
         if (to == this.cfg.node_id)
@@ -531,8 +617,4 @@ class AntiCluster
     }
 }
 
-AntiCluster.LEADER_ONLY = LEADER_ONLY;
-AntiCluster.NO_WAIT_QUORUM = NO_WAIT_QUORUM;
-AntiCluster.READ_FROM_FOLLOWER = READ_FROM_FOLLOWER;
-
 module.exports = AntiCluster;