antietcd 1.1.4 → 1.2.1

package/antietcd.js

@@ -12,12 +12,15 @@ const EventEmitter = require('events');
 
 const ws = require('ws');
 
+const TinyRaft = require('tinyraft');
+
 const EtcTree = require('./etctree.js');
 const AntiPersistence = require('./antipersistence.js');
 const AntiCluster = require('./anticluster.js');
+const AntiLocker = require('./antilocker.js');
 const { runCallbacks, de64, b64, RequestError } = require('./common.js');
 
-const VERSION = '1.1.4';
+const VERSION = '1.2.1';
 
 class AntiEtcd extends EventEmitter
 {
@@ -27,11 +30,27 @@ class AntiEtcd extends EventEmitter
         cfg['merge_watches'] = !('merge_watches' in cfg) || is_true(cfg['merge_watches']);
         cfg['stale_read'] = !('stale_read' in cfg) || is_true(cfg['stale_read']);
         cfg['use_base64'] = !('use_base64' in cfg) || is_true(cfg['use_base64']);
+        cfg['use_locks'] = cfg.cluster && (!('use_locks' in cfg) || is_true(cfg['use_locks']));
+        cfg['lock_wait_timeout'] = Number(cfg['lock_wait_timeout']);
+        if (!cfg['lock_wait_timeout'] || cfg['lock_wait_timeout'] < 1000)
+            cfg['lock_wait_timeout'] = 30000;
+        if (typeof cfg['logs'] == 'string')
+        {
+            const logs = {};
+            for (const k of cfg['logs'].split(/,/))
+                logs[k] = true;
+            cfg['logs'] = logs;
+        }
+        else if (!cfg['logs'])
+            cfg['logs'] = {};
+        else if (!(cfg['logs'] instanceof Object))
+            throw new Error('logs should be object or string with keywords (access,watch,cluster)');
         this.clients = {};
         this.client_id = 1;
-        this.etctree = new EtcTree(true);
+        this.etctree = new EtcTree();
         this.persistence = null;
         this.cluster = null;
+        this.locker = null;
         this.stored_term = 0;
         this.cfg = cfg;
         this.loading = false;
@@ -57,6 +76,11 @@ class AntiEtcd extends EventEmitter
         {
             this.cluster = new AntiCluster(this);
         }
+        if (this.cfg.use_locks)
+        {
+            this.locker = new AntiLocker(this.cfg);
+            this.locker.set_resume_notifications(rev => this.etctree.resume_notifications(rev));
+        }
         if (this.cfg.cert)
         {
             this.tls = {
@@ -103,6 +127,10 @@ class AntiEtcd extends EventEmitter
 
     async _persistAndReplicate(msg)
     {
+        if (this.cfg.use_locks && msg.events.length)
+        {
+            this.locker.set_txn_locks(msg.events, msg.header.revision);
+        }
         let res = [];
         if (this.cluster)
         {
@@ -144,6 +172,20 @@ class AntiEtcd extends EventEmitter
                 this.etctree.compact(revision);
             }
         }
+        if (this.cfg.use_locks && msg.events.length &&
+            (!this.cluster || this.cluster.raft.state == TinyRaft.LEADER))
+        {
+            // Leader unlocks txn itself, followers wait for feedback from the leader
+            this.locker.unlock_txn(msg.header.revision);
+            if (this.cluster)
+            {
+                this.cluster.unlockChange(msg.header.revision).catch(e =>
+                {
+                    if (!(e instanceof RequestError))
+                        console.error(e);
+                });
+            }
+        }
     }
 
     _handleRequest(req, res)
@@ -198,7 +240,7 @@ class AntiEtcd extends EventEmitter
             try
             {
                 // Access log
-                if (this.cfg.log_level > 1)
+                if (this.cfg.logs.access)
                 {
                     console.log(
                         new Date().toISOString()+
@@ -236,10 +278,10 @@ class AntiEtcd extends EventEmitter
             socket,
             alive: true,
             watches: {},
+            pinger: null,
         };
         socket.on('pong', () => this.clients[client_id].alive = true);
-        socket.on('error', e => console.error(e.syscall === 'connect' ? e.message : e));
-        const pinger = setInterval(() =>
+        const pinger = () =>
         {
             if (!this.clients[client_id])
             {
@@ -251,7 +293,18 @@ class AntiEtcd extends EventEmitter
             }
             this.clients[client_id].alive = false;
             socket.ping(() => {});
-        }, this.cfg.ws_keepalive_interval||30000);
+        };
+        if (socket.readyState == ws.WebSocket.OPEN)
+        {
+            this.clients[client_id].pinger = setInterval(pinger, this.cfg.ws_keepalive_interval||30000);
+        }
+        else
+        {
+            socket.on('open', () =>
+            {
+                this.clients[client_id].pinger = setInterval(pinger, this.cfg.ws_keepalive_interval||30000);
+            });
+        }
         socket.on('message', (msg) =>
         {
             try
@@ -272,17 +325,30 @@ class AntiEtcd extends EventEmitter
                 this._handleMessage(client_id, msg, socket);
             }
         });
-        socket.on('close', () =>
+        const on_close = () =>
         {
-            this._unsubscribeClient(client_id);
-            clearInterval(pinger);
-            delete this.clients[client_id];
+            if (this.clients[client_id])
+            {
+                this._unsubscribeClient(client_id);
+                if (this.clients[client_id].pinger)
+                {
+                    clearInterval(this.clients[client_id].pinger);
+                    this.clients[client_id].pinger = null;
+                }
+                delete this.clients[client_id];
+            }
             socket.terminate();
             if (reconnect)
             {
                 reconnect();
             }
+        };
+        socket.on('error', e =>
+        {
+            console.error(e.syscall === 'connect' ? e.message : e);
+            on_close();
         });
+        socket.on('close', on_close);
         return client_id;
     }
 
@@ -352,13 +418,7 @@ class AntiEtcd extends EventEmitter
         }
         if (this.cluster && path !== 'dump' && path != 'maintenance_status')
         {
-            const res = await this.cluster.checkRaftState(
-                path,
-                (data.leaderonly ? AntiCluster.LEADER_ONLY : 0) |
-                (data.serializable ? AntiCluster.READ_FROM_FOLLOWER : 0) |
-                (data.nowaitquorum ? AntiCluster.NO_WAIT_QUORUM : 0),
-                data
-            );
+            const res = await this.cluster.checkRaftState(path, data);
             if (res)
             {
                 return res;
@@ -416,13 +476,28 @@ class AntiEtcd extends EventEmitter
     // public watch API
     async create_watch(params, callback, stream_id)
     {
-        const watch = this.etctree.api_create_watch(this._encodeWatch(params), (msg) => callback(this._encodeMsg(msg)), stream_id);
+        params = this._encodeWatch(params);
+        if (this.cfg.use_locks && params.start_revision)
+        {
+            // watch is also a read, so it also has to follow read lock rules
+            const max_ts = Date.now() + this.cfg.lock_wait_timeout;
+            let lock_revision = 0;
+            while ((lock_revision = this.locker.check_locks(params.key, params.range_end)))
+            {
+                await this.locker.wait_commit(lock_revision, max_ts);
+            }
+        }
+        const watch = this.etctree.api_create_watch({ ...params, watch_id: null }, (msg) => callback(this._encodeMsg(msg)), stream_id);
         if (!watch.created)
         {
             throw new RequestError(400, 'Requested watch revision is compacted', { compact_revision: watch.compact_revision });
         }
         const watch_id = params.watch_id || watch.watch_id;
         this.api_watches[watch_id] = watch.watch_id;
+        if (watch.events)
+        {
+            callback(this._encodeMsg({ ...watch, watch_id }));
+        }
         return watch_id;
     }
 
@@ -440,30 +515,51 @@ class AntiEtcd extends EventEmitter
     // internal handlers
     async _handle_kv_txn(data)
     {
+        let decoded = data;
         if (this.cfg.use_base64)
         {
-            for (const item of data.compare||[])
+            decoded = {};
+            if (data.compare)
             {
-                if (item.key != null)
-                    item.key = de64(item.key);
+                decoded.compare = [];
+                for (const item of data.compare)
+                    decoded.compare.push({ ...item, key: de64(item.key) });
             }
-            for (const items of [ data.success, data.failure ])
+            for (const key of [ 'success', 'failure' ])
             {
-                for (const item of items||[])
+                if (!data[key])
+                    continue;
+                const actions = [];
+                for (const item of data[key])
                 {
-                    const req = item.request_range || item.requestRange ||
-                        item.request_put || item.requestPut ||
-                        item.request_delete_range || item.requestDeleteRange;
-                    if (req.key != null)
-                        req.key = de64(req.key);
-                    if (req.range_end != null)
-                        req.range_end = de64(req.range_end);
-                    if (req.value != null)
-                        req.value = de64(req.value);
+                    const copy = {};
+                    let r;
+                    if ((r = (item.request_range || item.requestRange)))
+                        copy.request_range = { ...r, key: de64(r.key), range_end: de64(r.range_end) };
+                    else if ((r = (item.request_delete_range || item.requestDeleteRange)))
+                        copy.request_delete_range = { ...r, key: de64(r.key), range_end: de64(r.range_end) };
+                    else if ((r = (item.request_put || item.requestPut)))
+                        copy.request_put = { ...r, key: de64(r.key), value: de64(r.value) };
+                    actions.push(copy);
                 }
+                decoded[key] = actions;
             }
         }
-        const result = await this.etctree.api_txn(data);
+        if (this.cfg.use_locks)
+        {
+            const max_ts = Date.now() + this.cfg.lock_wait_timeout;
+            let lock_revision = 0;
+            while ((lock_revision = this.locker.check_txn_locks(decoded)))
+            {
+                await this.locker.wait_commit(lock_revision, max_ts);
+                const res = await this.cluster.checkRaftState('kv_txn', data);
+                if (res)
+                {
+                    return res;
+                }
+            }
+        }
+        const result = await this.etctree.api_txn(decoded);
         if (this.cfg.use_base64)
         {
             for (const item of result.responses||[])
@@ -550,28 +646,23 @@ class AntiEtcd extends EventEmitter
     _handleMessage(client_id, msg, socket)
     {
         const client = this.clients[client_id];
-        if (this.cfg.access_log)
+        if (this.cfg.logs.access)
         {
             console.log(new Date().toISOString()+' '+client.addr+' '+(client.raft_node_id || '-')+' -> '+JSON.stringify(msg));
         }
         if (msg.create_request)
         {
-            const create_request = msg.create_request;
+            const create_request = this._encodeWatch(msg.create_request);
             if (!create_request.watch_id || !client.watches[create_request.watch_id])
             {
-                client.send_cb = client.send_cb || (msg => socket.send(JSON.stringify(this._encodeMsg(msg))));
-                const watch = this.etctree.api_create_watch(
-                    this._encodeWatch(create_request), client.send_cb, (this.cfg.merge_watches ? 'C'+client_id : null)
-                );
-                if (!watch.created)
+                client.send_cb = client.send_cb || (msg => socket.send(JSON.stringify(this._encodeMsg(msg, client))));
+                if (this.cfg.use_locks && create_request.start_revision)
                 {
-                    socket.send(JSON.stringify({ result: { header: { revision: this.etctree.mod_revision }, watch_id: create_request.watch_id, ...watch } }));
+                    this._lock_and_create_watch(client, create_request);
                 }
                 else
                 {
-                    create_request.watch_id = create_request.watch_id || watch.watch_id;
-                    client.watches[create_request.watch_id] = watch.watch_id;
-                    socket.send(JSON.stringify({ result: { header: { revision: this.etctree.mod_revision }, watch_id: create_request.watch_id, created: true } }));
+                    this._create_client_watch(client, create_request);
                 }
             }
         }
@@ -599,21 +690,76 @@ class AntiEtcd extends EventEmitter
         }
     }
 
+    async _lock_and_create_watch(client, create_request)
+    {
+        // watch is also a read, so it also has to follow read lock rules
+        try
+        {
+            const max_ts = Date.now() + this.cfg.lock_wait_timeout;
+            let lock_revision = 0;
+            while ((lock_revision = this.locker.check_locks(create_request.key, create_request.range_end)))
+            {
+                await this.locker.wait_commit(lock_revision, max_ts);
+            }
+            this._create_client_watch(client, create_request);
+        }
+        catch (e)
+        {
+            if (!(e instanceof RequestError))
+            {
+                console.error(e);
+            }
+            client.send_cb({ result: {
+                header: { revision: this.etctree.mod_revision },
+                watch_id: create_request.watch_id,
+                canceled: true,
+                cancel_reason: e.message,
+            } });
+        }
+    }
+
+    _create_client_watch(client, create_request)
+    {
+        const watch = this.etctree.api_create_watch(
+            { ...create_request, watch_id: null }, client.send_cb, (this.cfg.merge_watches ? 'C'+client.id : null)
+        );
+        if (!watch.created)
+        {
+            client.send_cb({ result: { header: { revision: this.etctree.mod_revision }, watch_id: create_request.watch_id, ...watch } });
+        }
+        else
+        {
+            create_request.watch_id = create_request.watch_id || watch.watch_id;
+            client.watches[create_request.watch_id] = watch.watch_id;
+            client.send_cb({ result: {
+                header: { revision: this.etctree.mod_revision },
+                watch_id: create_request.watch_id,
+                created: true,
+                events: watch.events,
+            }});
+        }
+    }
+
     _encodeWatch(create_request)
     {
-        const req = { ...create_request, watch_id: null };
         if (this.cfg.use_base64)
         {
+            const req = { ...create_request };
             if (req.key != null)
                 req.key = de64(req.key);
             if (req.range_end != null)
                 req.range_end = de64(req.range_end);
+            return req;
         }
-        return req;
+        return create_request;
     }
 
-    _encodeMsg(msg)
+    _encodeMsg(msg, client)
     {
+        if (this.cfg.logs.watch && client)
+        {
+            console.log(new Date().toISOString()+' '+client.addr+' '+(client.raft_node_id || '-')+' <- '+JSON.stringify(msg));
+        }
         if (this.cfg.use_base64 && msg.result && msg.result.events)
         {
             return { ...msg, result: { ...msg.result, events: msg.result.events.map(ev => ({

package/antilocker.js

@@ -0,0 +1,212 @@
+// "Row" (key)-level locks for antietcd
+// (c) Vitaliy Filippov, 2024+
+// License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
+
+const { RequestError } = require('./common.js');
+
+// How does it work:
+// - Leader sets key-level locks, persists and replicates the change, then removes locks
+// - Followers set locks, persist and reply to the leader, wait for the feedback, then remove locks
+// - This is done from etctree by calling regular 'replicate watcher' method so etctree is agnostic of it
+// - Both leader and followers check for locks with check_txn_locks() and then wait for them
+//   using wait_commit() before executing the transaction, and only proceed when check_txn_locks() returns 0
+// - A node calls break_locks() if their Raft state changes
+class AntiLocker
+{
+    constructor(cfg)
+    {
+        cfg = cfg || {};
+        this.cfg = cfg;
+        this.cfg.lock_wait_interval = Number(this.cfg.lock_wait_interval);
+        if (!this.cfg.lock_wait_interval || this.cfg.lock_wait_interval < 50)
+            this.cfg.lock_wait_interval = 3000;
+        this.locks = [];
+        this.on_unlock = [];
+        this.txn_locks = {};
+        this.lock_timeout_timer = this.cfg.use_locks ? setInterval(() => this._timeout_locks(), this.cfg.lock_wait_interval) : null;
+        this.resume_notifications = null;
+    }
+
+    destroy()
+    {
+        if (this.lock_timeout_timer)
+        {
+            clearInterval(this.lock_timeout_timer);
+            this.lock_timeout_timer = null;
+        }
+    }
+
+    // Put (txn) => etctree.resume_notifications(txn) here
+    set_resume_notifications(resume_cb)
+    {
+        this.resume_notifications = resume_cb;
+    }
+
+    _find_lock(key)
+    {
+        let min = 0, max = this.locks.length;
+        while (max-min > 1)
+        {
+            let mid = Math.floor((min+max)/2);
+            if (this.locks[mid].key > key)
+                max = mid;
+            else
+                min = mid;
+        }
+        return min;
+    }
+
+    check_locks(key, range_end)
+    {
+        const pos = this._find_lock(key);
+        if (pos < this.locks.length && this.locks[pos].key <= (key || range_end))
+        {
+            return this.locks[pos].revision;
+        }
+        return 0;
+    }
+
+    check_txn_locks(txn)
+    {
+        let lock_txn;
+        if (txn.compare)
+        {
+            for (const req of txn.compare)
+            {
+                lock_txn = this.check_locks(req.key);
+                if (lock_txn)
+                    return lock_txn;
+            }
+        }
+        for (const actions of [ 'success', 'failure' ])
+        {
+            if (!txn[actions])
+                continue;
+            let r, lock_txn;
+            for (const req of txn[actions])
+            {
+                if ((r = (req.request_range || req.requestRange)))
+                {
+                    lock_txn = this.check_locks(r.key, r.range_end);
+                    if (lock_txn)
+                        return lock_txn;
+                }
+                else if ((r = (req.request_put || req.requestPut)))
+                {
+                    lock_txn = this.check_locks(r.key);
+                    if (lock_txn)
+                        return lock_txn;
+                }
+                else if ((r = (req.request_delete_range || req.requestDeleteRange)))
+                {
+                    lock_txn = this.check_locks(r.key, r.range_end);
+                    if (lock_txn)
+                        return lock_txn;
+                }
+            }
+        }
+        return 0;
+    }
+
+    set_txn_locks(notifications, mod_revision)
+    {
+        if (this.txn_locks[mod_revision])
+        {
+            throw new Error('BUG: locks already set for revision '+mod_revision);
+        }
+        const res = [];
+        for (const event of notifications)
+        {
+            const pos = this._find_lock(event.key);
+            const lock = { key: event.key, revision: mod_revision };
+            res.push(lock);
+            this.locks.splice(pos, 0, lock);
+        }
+        this.txn_locks[mod_revision] = res;
+    }
+
+    wait_commit(mod_revision, expire_ts)
+    {
+        // Postpone transaction
+        return new Promise((ok, fail) => this._set_on_unlock(mod_revision, expire_ts, ok, fail));
+    }
+
+    break_locks()
+    {
+        this.txn_locks = {};
+        this.locks = [];
+        const waits = this.on_unlock;
+        this.on_unlock = [];
+        for (const wait of waits)
+        {
+            const cb = wait.fail;
+            cb(new RequestError(408, 'Lock wait aborted, please retry request'));
+        }
+    }
+
+    _set_on_unlock(lock_rev, expire_ts, ok, fail)
+    {
+        let min = 0, max = this.on_unlock.length;
+        while (max-min > 1)
+        {
+            let mid = Math.floor((min+max)/2);
+            if (this.on_unlock[mid].revision > lock_rev)
+                max = mid;
+            else
+                min = mid;
+        }
+        this.on_unlock.splice(min, 0, { revision: lock_rev, expire_ts, ok, fail });
+    }
+
+    _timeout_locks()
+    {
+        const now = Date.now();
+        const cancel = [];
+        for (let i = 0; i < this.on_unlock.length; i++)
+        {
+            if (now > this.on_unlock[i].expire_ts)
+            {
+                cancel.push(this.on_unlock[i]);
+                this.on_unlock.splice(i, 1);
+                i--;
+            }
+        }
+        for (const wait of cancel)
+        {
+            const cb = wait.fail;
+            cb(new RequestError(408, 'Lock wait timeout, please retry request'));
+        }
+    }
+
+    unlock_txn(lock_rev)
+    {
+        if (!this.txn_locks[lock_rev])
+        {
+            throw new Error('BUG: revision '+lock_rev+' not locked');
+        }
+        const txn_set = new Set();
+        for (const lock of this.txn_locks[lock_rev])
+        {
+            txn_set.add(lock);
+        }
+        delete this.txn_locks[lock_rev];
+        this.locks = this.locks.filter(l => !txn_set.has(l));
+        let i = 0;
+        while (i < this.on_unlock.length && this.on_unlock[i].revision <= lock_rev)
+        {
+            i++;
+        }
+        const waits = this.on_unlock.splice(0, i);
+        for (const wait of waits)
+        {
+            const cb = wait.ok;
+            cb();
+        }
+        if (this.resume_notifications)
+        {
+            this.resume_notifications(lock_rev);
+        }
+    }
+}
+
+module.exports = AntiLocker;