hungry-ghost-hive 0.44.0 → 0.46.0

package/dist/cluster/partition-safety.test.js

@@ -0,0 +1,440 @@
+// Licensed under the Hungry Ghost Hive License. See LICENSE.
+import { mkdirSync, mkdtempSync, rmSync } from 'fs';
+import { createServer as createNetServer } from 'net';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { ClusterRuntime } from './runtime.js';
+const tempRoots = [];
+const activeRuntimes = [];
+afterEach(async () => {
+    for (const runtime of activeRuntimes.splice(0)) {
+        try {
+            await runtime.stop();
+        }
+        catch {
+            // Best effort shutdown for test cleanup.
+        }
+    }
+    for (const root of tempRoots.splice(0)) {
+        rmSync(root, { recursive: true, force: true });
+    }
+});
+describe('fencing token validation', () => {
+    it('rejects heartbeats with fencing_token lower than term', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-fence-reject',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        // First, advance the term by accepting a vote request
+        await postJson(fixture.config.public_url, '/cluster/v1/election/request-vote', {
+            term: 5,
+            candidate_id: 'candidate-5',
+        });
+        // Send heartbeat with valid term but stale fencing token
+        const res = await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 5,
+            leader_id: 'candidate-5',
+            fencing_token: 3,
+        });
+        expect(res.success).toBe(false);
+        expect(res.fencing_token).toBe(5);
+    });
+    it('accepts heartbeats with valid fencing_token', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-fence-accept',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        const res = await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 3,
+            leader_id: 'leader-3',
+            fencing_token: 3,
+        });
+        expect(res.success).toBe(true);
+        expect(res.fencing_token).toBe(3);
+    });
+    it('rejects delta requests with stale fencing_token', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-delta-fence',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        // Advance term
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 10,
+            leader_id: 'leader-10',
+            fencing_token: 10,
+        });
+        // Request delta with stale fencing token
+        const res = await fetch(`${fixture.config.public_url}/cluster/v1/events/delta`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                version_vector: {},
+                fencing_token: 5,
+            }),
+        });
+        expect(res.status).toBe(409);
+        const body = (await res.json());
+        expect(body.error).toContain('stale leader epoch');
+        expect(body.fencing_token).toBe(10);
+    });
+    it('accepts delta requests with current fencing_token', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-delta-fence-ok',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        // Set term to 4
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 4,
+            leader_id: 'leader-4',
+            fencing_token: 4,
+        });
+        // Request delta with matching fencing token
+        const res = await fetch(`${fixture.config.public_url}/cluster/v1/events/delta`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                version_vector: {},
+                fencing_token: 4,
+            }),
+        });
+        expect(res.status).toBe(200);
+        const body = (await res.json());
+        expect(body.fencing_token).toBe(4);
+    });
+    it('accepts delta requests without fencing_token for backward compatibility', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-delta-no-fence',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        const res = await fetch(`${fixture.config.public_url}/cluster/v1/events/delta`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                version_vector: {},
+            }),
+        });
+        expect(res.status).toBe(200);
+    });
+    it('returns fencing_token in status endpoint', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-status-fence',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        // Advance term
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 7,
+            leader_id: 'leader-7',
+            fencing_token: 7,
+        });
+        const status = fixture.runtime.getStatus();
+        expect(status.fencing_token).toBe(7);
+        expect(status.term).toBe(7);
+    });
+});
+describe('leader lease validation', () => {
+    it('reports lease invalid when no heartbeat has been received', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-none',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+        });
+        const status = fixture.runtime.getStatus();
+        expect(status.leader_lease_valid).toBe(false);
+    });
+    it('reports lease valid immediately after receiving heartbeat', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-fresh',
+            election_timeout_min_ms: 2000,
+            election_timeout_max_ms: 2000,
+            heartbeat_interval_ms: 100,
+        });
+        // Send a heartbeat
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 2,
+            leader_id: 'leader-2',
+            fencing_token: 2,
+        });
+        const status = fixture.runtime.getStatus();
+        expect(status.leader_lease_valid).toBe(true);
+    });
+    it('leader always reports lease valid', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-leader',
+            election_timeout_min_ms: 80,
+            election_timeout_max_ms: 120,
+            heartbeat_interval_ms: 60,
+        });
+        await waitFor(() => fixture.runtime.getStatus().is_leader, 4000);
+        const status = fixture.runtime.getStatus();
+        expect(status.leader_lease_valid).toBe(true);
+    });
+    it('reports lease expired after timeout elapses without heartbeat', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const leaseMs = 150;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-expire',
+            election_timeout_min_ms: 5000,
+            election_timeout_max_ms: 5000,
+            heartbeat_interval_ms: 50,
+            leader_lease_ms: leaseMs,
+        });
+        // Send heartbeat to establish lease
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 1,
+            leader_id: 'leader-1',
+            fencing_token: 1,
+        });
+        expect(fixture.runtime.getStatus().leader_lease_valid).toBe(true);
+        // Wait for lease to expire
+        await new Promise(resolve => setTimeout(resolve, leaseMs + 50));
+        expect(fixture.runtime.getStatus().leader_lease_valid).toBe(false);
+    });
+    it('reports correct leader_lease_duration_ms from config', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-config',
+            heartbeat_interval_ms: 200,
+            leader_lease_ms: 1000,
+        });
+        expect(fixture.runtime.getStatus().leader_lease_duration_ms).toBe(1000);
+    });
+    it('defaults leader_lease_duration_ms to 3x heartbeat_interval_ms', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-default',
+            heartbeat_interval_ms: 200,
+        });
+        expect(fixture.runtime.getStatus().leader_lease_duration_ms).toBe(600);
+    });
+    it('resets lease on step-down from higher term', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-lease-stepdown',
+            election_timeout_min_ms: 5000,
+            election_timeout_max_ms: 5000,
+            heartbeat_interval_ms: 100,
+        });
+        // Establish lease at term 2
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 2,
+            leader_id: 'leader-2',
+            fencing_token: 2,
+        });
+        expect(fixture.runtime.getStatus().leader_lease_valid).toBe(true);
+        // Higher term vote request causes step-down, which should reset lease
+        await postJson(fixture.config.public_url, '/cluster/v1/election/request-vote', {
+            term: 5,
+            candidate_id: 'candidate-5',
+        });
+        // Lease should be invalid after step-down (no heartbeat from new leader yet)
+        expect(fixture.runtime.getStatus().leader_lease_valid).toBe(false);
+    });
+});
+describe('partition healing scenarios', () => {
+    it('stale leader is fenced after partition heals', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const portA = await getFreePort();
+        const portB = await getFreePort();
+        // Node A and B are peers
+        const configA = await buildConfig({
+            node_id: 'node-a-heal',
+            listen_port: portA,
+            public_url: `http://127.0.0.1:${portA}`,
+            peers: [{ id: 'node-b-heal', url: `http://127.0.0.1:${portB}` }],
+            election_timeout_min_ms: 80,
+            election_timeout_max_ms: 120,
+            heartbeat_interval_ms: 60,
+        });
+        const configB = await buildConfig({
+            node_id: 'node-b-heal',
+            listen_port: portB,
+            public_url: `http://127.0.0.1:${portB}`,
+            peers: [{ id: 'node-a-heal', url: `http://127.0.0.1:${portA}` }],
+            election_timeout_min_ms: 80,
+            election_timeout_max_ms: 120,
+            heartbeat_interval_ms: 60,
+        });
+        const fixtureA = await startRuntimeWithConfig(configA);
+        const fixtureB = await startRuntimeWithConfig(configB);
+        // Wait until at least one becomes leader
+        await waitFor(() => fixtureA.runtime.getStatus().is_leader || fixtureB.runtime.getStatus().is_leader, 4000);
+        const statusA = fixtureA.runtime.getStatus();
+        const statusB = fixtureB.runtime.getStatus();
+        // Exactly one should be leader (same term wins in a 2-node cluster)
+        const leaderCount = [statusA, statusB].filter(s => s.is_leader).length;
+        expect(leaderCount).toBeLessThanOrEqual(1);
+        // Both should have fencing tokens
+        expect(statusA.fencing_token).toBeGreaterThanOrEqual(0);
+        expect(statusB.fencing_token).toBeGreaterThanOrEqual(0);
+    });
+    it('follower rejects stale leader heartbeat after seeing higher term', async () => {
+        if (!(await canListenOnLocalhost()))
+            return;
+        const fixture = await startRuntimeFixture({
+            node_id: 'node-heal-reject',
+            election_timeout_min_ms: 5000,
+            election_timeout_max_ms: 5000,
+        });
+        // Node sees term 10 from new leader
+        await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 10,
+            leader_id: 'new-leader',
+            fencing_token: 10,
+        });
+        // Old leader (term 5) tries to send heartbeat after partition heals
+        const staleRes = await postJson(fixture.config.public_url, '/cluster/v1/election/heartbeat', {
+            term: 5,
+            leader_id: 'old-leader',
+            fencing_token: 5,
+        });
+        expect(staleRes.success).toBe(false);
+        expect(staleRes.fencing_token).toBe(10);
+        // Verify node still follows new leader
+        const status = fixture.runtime.getStatus();
+        expect(status.leader_id).toBe('new-leader');
+        expect(status.term).toBe(10);
+    });
+});
+// --- Test helpers ---
+async function startRuntimeFixture(overrides = {}) {
+    const attempts = overrides.listen_port ? 1 : 5;
+    let lastError;
+    for (let i = 0; i < attempts; i++) {
+        const config = await buildConfig(overrides);
+        try {
+            return await startRuntimeWithConfig(config);
+        }
+        catch (error) {
+            lastError = error;
+            const err = error;
+            if (!overrides.listen_port && err.code === 'EADDRINUSE') {
+                continue;
+            }
+            throw error;
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error('Failed to start runtime fixture');
+}
+async function startRuntimeWithConfig(config) {
+    const root = mkdtempSync(join(tmpdir(), `hive-partition-safety-${config.node_id}-`));
+    const hiveDir = join(root, '.hive');
+    mkdirSync(hiveDir, { recursive: true });
+    const runtime = new ClusterRuntime(config, { hiveDir });
+    try {
+        await runtime.start();
+        activeRuntimes.push(runtime);
+        tempRoots.push(root);
+        return { root, hiveDir, config, runtime };
+    }
+    catch (error) {
+        try {
+            await runtime.stop();
+        }
+        catch {
+            // Best effort cleanup for partial starts.
+        }
+        rmSync(root, { recursive: true, force: true });
+        throw error;
+    }
+}
+async function buildConfig(overrides = {}) {
+    const port = overrides.listen_port ?? (await getFreePort());
+    const base = {
+        enabled: true,
+        node_id: 'node-test',
+        listen_host: '127.0.0.1',
+        listen_port: port,
+        public_url: `http://127.0.0.1:${port}`,
+        peers: [],
+        heartbeat_interval_ms: 100,
+        election_timeout_min_ms: 150,
+        election_timeout_max_ms: 250,
+        sync_interval_ms: 200,
+        request_timeout_ms: 600,
+        story_similarity_threshold: 0.8,
+    };
+    return {
+        ...base,
+        ...overrides,
+        public_url: overrides.public_url || base.public_url,
+        peers: overrides.peers || base.peers,
+    };
+}
+async function postJson(baseUrl, path, body) {
+    const res = await fetch(`${baseUrl}${path}`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(body),
+    });
+    return (await res.json());
+}
+async function waitFor(predicate, timeoutMs) {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+        if (predicate())
+            return;
+        await new Promise(resolve => setTimeout(resolve, 25));
+    }
+    throw new Error('Timed out waiting for condition');
+}
+async function getFreePort() {
+    return new Promise((resolve, reject) => {
+        const server = createNetServer();
+        server.once('error', reject);
+        server.listen(0, '127.0.0.1', () => {
+            const address = server.address();
+            if (!address || typeof address === 'string') {
+                server.close(() => reject(new Error('Failed to allocate free port')));
+                return;
+            }
+            const port = address.port;
+            server.close(err => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+                resolve(port);
+            });
+        });
+    });
+}
+async function canListenOnLocalhost() {
+    try {
+        await getFreePort();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=partition-safety.test.js.map

package/dist/cluster/partition-safety.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"partition-safety.test.js","sourceRoot":"","sources":["../../src/cluster/partition-safety.test.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AACpD,OAAO,EAAE,YAAY,IAAI,eAAe,EAAE,MAAM,KAAK,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAS9C,MAAM,SAAS,GAAa,EAAE,CAAC;AAC/B,MAAM,cAAc,GAAqB,EAAE,CAAC;AAE5C,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,KAAK,MAAM,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,yCAAyC;QAC3C,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,sDAAsD;QACtD,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,mCAAmC,EAAE;YAC7E,IAAI,EAAE,CAAC;YACP,YAAY,EAAE,aAAa;SAC5B,CAAC,CAAC;QAEH,yDAAyD;QACzD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YACtF,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,aAAa;YACxB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YACtF,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,kBAAkB;YAC3B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,eAAe;QACf,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,WAAW;YACtB,aAAa,EAAE,EAAE;SAClB,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,0BAA0B,EAAE;YAC9E,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,cAAc,EAAE,EAAE;gBAClB,aAAa,EAAE,CAAC;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6C,CAAC;QAC5E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,qBAAqB;YAC9B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,gBAAgB;QAChB,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,4CAA4C;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,0BAA0B,EAAE;YAC9E,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,cAAc,EAAE,EAAE;gBAClB,aAAa,EAAE,CAAC;aACjB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA8B,CAAC;QAC7D,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;QACvF,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,qBAAqB;YAC9B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,0BAA0B,EAAE;YAC9E,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,cAAc,EAAE,EAAE;aACnB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,eAAe;QACf,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,iBAAiB;YAC1B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,kBAAkB;YAC3B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;YAC7B,qBAAqB,EAAE,GAAG;SAC3B,CAAC,CAAC;QAEH,mBAAmB;QACnB,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,uBAAuB,EAAE,EAAE;YAC3B,uBAAuB,EAAE,GAAG;YAC5B,qBAAqB,EAAE,EAAE;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,GAAG,CAAC;QACpB,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;YAC7B,qBAAqB,EAAE,EAAE;YACzB,eAAe,EAAE,OAAO;SACzB,CAAC,CAAC;QAEH,oCAAoC;QACpC,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElE,2BAA2B;QAC3B,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;QAEhE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,mBAAmB;YAC5B,qBAAqB,EAAE,GAAG;YAC1B,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,oBAAoB;YAC7B,qBAAqB,EAAE,GAAG;SAC3B,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,qBAAqB;YAC9B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;YAC7B,qBAAqB,EAAE,GAAG;SAC3B,CAAC,CAAC;QAEH,4BAA4B;QAC5B,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,UAAU;YACrB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElE,sEAAsE;QACtE,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,mCAAmC,EAAE;YAC7E,IAAI,EAAE,CAAC;YACP,YAAY,EAAE,aAAa;SAC5B,CAAC,CAAC;QAEH,6EAA6E;QAC7E,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,KAAK,GAAG,MAAM,WAAW,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,WAAW,EAAE,CAAC;QAElC,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,KAAK;YAClB,UAAU,EAAE,oBAAoB,KAAK,EAAE;YACvC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,oBAAoB,KAAK,EAAE,EAAE,CAAC;YAChE,uBAAuB,EAAE,EAAE;YAC3B,uBAAuB,EAAE,GAAG;YAC5B,qBAAqB,EAAE,EAAE;SAC1B,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,OAAO,EAAE,aAAa;YACtB,WAAW,EAAE,KAAK;YAClB,UAAU,EAAE,oBAAoB,KAAK,EAAE;YACvC,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,oBAAoB,KAAK,EAAE,EAAE,CAAC;YAChE,uBAAuB,EAAE,EAAE;YAC3B,uBAAuB,EAAE,GAAG;YAC5B,qBAAqB,EAAE,EAAE;SAC1B,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;QAEvD,yCAAyC;QACzC,MAAM,OAAO,CACX,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,SAAS,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,SAAS,EACtF,IAAI,CACL,CAAC;QAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAE7C,oEAAoE;QACpE,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;QACvE,MAAM,CAAC,WAAW,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAE3C,kCAAkC;QAClC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAChF,IAAI,CAAC,CAAC,MAAM,oBAAoB,EAAE,CAAC;YAAE,OAAO;QAE5C,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC;YACxC,OAAO,EAAE,kBAAkB;YAC3B,uBAAuB,EAAE,IAAI;YAC7B,uBAAuB,EAAE,IAAI;SAC9B,CAAC,CAAC;QAEH,oCAAoC;QACpC,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC1E,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,YAAY;YACvB,aAAa,EAAE,EAAE;SAClB,CAAC,CAAC;QAEH,oEAAoE;QACpE,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,gCAAgC,EAAE;YAC3F,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,YAAY;YACvB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;QAEH,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAExC,uCAAuC;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,uBAAuB;AAEvB,KAAK,UAAU,mBAAmB,CAChC,YAAoC,EAAE;IAEtC,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC;YACH,OAAO,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,GAAG,GAAG,KAA8B,CAAC;YAC3C,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACxD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;AAC9F,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,MAAqB;IACzD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,yBAAyB,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;QACD,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,YAAoC,EAAE;IAC/D,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,IAAI,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAkB;QAC1B,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,WAAW;QACxB,WAAW,EAAE,IAAI;QACjB,UAAU,EAAE,oBAAoB,IAAI,EAAE;QACtC,KAAK,EAAE,EAAE;QACT,qBAAqB,EAAE,GAAG;QAC1B,uBAAuB,EAAE,GAAG;QAC5B,uBAAuB,EAAE,GAAG;QAC5B,gBAAgB,EAAE,GAAG;QACrB,kBAAkB,EAAE,GAAG;QACvB,0BAA0B,EAAE,GAAG;KAChC,CAAC;IAEF,OAAO;QACL,GAAG,IAAI;QACP,GAAG,SAAS;QACZ,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU;QACnD,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK;KACrC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,OAAe,EACf,IAAY,EACZ,IAA6B;IAE7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,EAAE;QAC3C,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;KAC3B,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAwB,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,SAAwB,EAAE,SAAiB;IAChE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;QACtC,IAAI,SAAS,EAAE;YAAE,OAAO;QACxB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,WAAW;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE;YACjC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBAC5C,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC;gBACtE,OAAO;YACT,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;gBACjB,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,GAAG,CAAC,CAAC;oBACZ,OAAO;gBACT,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,oBAAoB;IACjC,IAAI,CAAC;QACH,MAAM,WAAW,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/cluster/raft-state-machine.d.ts

@@ -1,6 +1,7 @@
 import type { ClusterConfig, ClusterPeerConfig } from '../config/schema.js';
-import type { DurableLogEntryType } from './raft-store.js';
+import type { CompactionResult, DurableLogEntryType } from './raft-store.js';
 import { RaftMetadataStore } from './raft-store.js';
+import type { VersionVector } from './types.js';
 type NodeRole = 'leader' | 'follower' | 'candidate';
 interface VoteResponse {
     term: number;
@@ -19,11 +20,40 @@ export declare class RaftStateMachine {
     currentTerm: number;
     votedFor: string | null;
     leaderId: string | null;
+    lastHeartbeatReceivedAt: number;
+    /**
+     * When true, this node is catching up from a snapshot and must not
+     * participate in leader elections until fully recovered.
+     */
+    isCatchingUp: boolean;
+    /** Dynamic peer list that can be updated at runtime via membership changes. */
+    private dynamicPeers;
     private electionDeadline;
     private electionInFlight;
     private electionTimer;
     private raftStore;
     constructor(config: ClusterConfig, deps: RaftStateMachineDeps);
+    /** Returns the active peer list (dynamic if set, otherwise static config). */
+    getPeers(): ClusterPeerConfig[];
+    /** Replaces the dynamic peer list. */
+    setPeers(peers: ClusterPeerConfig[]): void;
+    /** Returns the leader lease window in milliseconds. */
+    get leaderLeaseDurationMs(): number;
+    /**
+     * Returns true when this follower has received a valid heartbeat
+     * from the current leader within the lease window.
+     */
+    isLeaderLeaseValid(): boolean;
+    /**
+     * The fencing token is the current Raft term. Operations tagged with a
+     * lower term than ours must be rejected to prevent stale-leader writes.
+     */
+    getFencingToken(): number;
+    /**
+     * Validates a fencing token from a remote node. Returns true when the
+     * token is at least as recent as our current term.
+     */
+    validateFencingToken(token: number): boolean;
     initializeRaftStore(hiveDir: string): void;
     getRaftStore(): RaftMetadataStore | null;
     clearRaftStore(): void;
@@ -42,6 +72,8 @@ export declare class RaftStateMachine {
         last_applied: number;
         last_log_index: number;
     } | null;
+    getLogEntryCount(): number;
+    createSnapshotAndCompact(versionVector: VersionVector): CompactionResult;
     getLeaderUrl(): string | null;
 }
 export {};

package/dist/cluster/raft-state-machine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"raft-state-machine.d.ts","sourceRoot":"","sources":["../../src/cluster/raft-state-machine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,KAAK,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;AAOpD,UAAU,YAAY;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACzF,QAAQ,EAAE,MAAM,OAAO,CAAC;IACxB,qBAAqB,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACjD;AAED,qBAAa,gBAAgB;IAYzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAZvB,IAAI,EAAE,QAAQ,CAAc;IAC5B,WAAW,SAAK;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAQ;IAC/B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAQ;IAE/B,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,SAAS,CAAkC;gBAGhC,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,oBAAoB;IAG7C,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAgB1C,YAAY,IAAI,iBAAiB,GAAG,IAAI;IAIxC,cAAc,IAAI,IAAI;IAItB,iBAAiB,IAAI,IAAI;IAYzB,gBAAgB,IAAI,IAAI;IAQlB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAuEpC,iBAAiB,CAAC,IAAI,EAAE,OAAO,GAAG,YAAY;IAgC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAmBrD,MAAM,IAAI,MAAM;IAKhB,qBAAqB,IAAI,IAAI;IAQ7B,gBAAgB,IAAI,IAAI;IAUxB,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAYtF,+BAA+B,CAAC,MAAM,EAAE,OAAO,kBAAkB,EAAE,YAAY,EAAE,GAAG,MAAM;IAQ1F,iBAAiB,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI;IAIR,YAAY,IAAI,MAAM,GAAG,IAAI;CAO9B"}
+{"version":3,"file":"raft-state-machine.d.ts","sourceRoot":"","sources":["../../src/cluster/raft-state-machine.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,KAAK,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,KAAK,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;AAOpD,UAAU,YAAY;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACzF,QAAQ,EAAE,MAAM,OAAO,CAAC;IACxB,qBAAqB,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACjD;AAED,qBAAa,gBAAgB;IAsBzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAtBvB,IAAI,EAAE,QAAQ,CAAc;IAC5B,WAAW,SAAK;IAChB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAQ;IAC/B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAQ;IAC/B,uBAAuB,SAAK;IAE5B;;;OAGG;IACH,YAAY,UAAS;IAErB,+EAA+E;IAC/E,OAAO,CAAC,YAAY,CAAoC;IAExD,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,SAAS,CAAkC;gBAGhC,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,oBAAoB;IAG7C,8EAA8E;IAC9E,QAAQ,IAAI,iBAAiB,EAAE;IAI/B,sCAAsC;IACtC,QAAQ,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,IAAI;IAI1C,uDAAuD;IACvD,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED;;;OAGG;IACH,kBAAkB,IAAI,OAAO;IAM7B;;;OAGG;IACH,eAAe,IAAI,MAAM;IAIzB;;;OAGG;IACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAI5C,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAgB1C,YAAY,IAAI,iBAAiB,GAAG,IAAI;IAIxC,cAAc,IAAI,IAAI;IAItB,iBAAiB,IAAI,IAAI;IAkBzB,gBAAgB,IAAI,IAAI;IAQlB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAuEpC,iBAAiB,CAAC,IAAI,EAAE,OAAO,GAAG,YAAY;IAgC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAoBrD,MAAM,IAAI,MAAM;IAKhB,qBAAqB,IAAI,IAAI;IAQ7B,gBAAgB,IAAI,IAAI;IAUxB,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAYtF,+BAA+B,CAAC,MAAM,EAAE,OAAO,kBAAkB,EAAE,YAAY,EAAE,GAAG,MAAM;IAQ1F,iBAAiB,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI;IAIR,gBAAgB,IAAI,MAAM;IAI1B,wBAAwB,CAAC,aAAa,EAAE,aAAa,GAAG,gBAAgB;IASxE,YAAY,IAAI,MAAM,GAAG,IAAI;CAO9B"}

package/dist/cluster/raft-state-machine.js

@@ -8,6 +8,14 @@ export class RaftStateMachine {
     currentTerm = 0;
     votedFor = null;
     leaderId = null;
+    lastHeartbeatReceivedAt = 0;
+    /**
+     * When true, this node is catching up from a snapshot and must not
+     * participate in leader elections until fully recovered.
+     */
+    isCatchingUp = false;
+    /** Dynamic peer list that can be updated at runtime via membership changes. */
+    dynamicPeers = null;
     electionDeadline = 0;
     electionInFlight = false;
     electionTimer = null;
@@ -16,6 +24,43 @@ export class RaftStateMachine {
         this.config = config;
         this.deps = deps;
     }
+    /** Returns the active peer list (dynamic if set, otherwise static config). */
+    getPeers() {
+        return this.dynamicPeers ?? this.config.peers;
+    }
+    /** Replaces the dynamic peer list. */
+    setPeers(peers) {
+        this.dynamicPeers = peers;
+    }
+    /** Returns the leader lease window in milliseconds. */
+    get leaderLeaseDurationMs() {
+        return this.config.leader_lease_ms ?? this.config.heartbeat_interval_ms * 3;
+    }
+    /**
+     * Returns true when this follower has received a valid heartbeat
+     * from the current leader within the lease window.
+     */
+    isLeaderLeaseValid() {
+        if (this.role === 'leader')
+            return true;
+        if (this.lastHeartbeatReceivedAt === 0)
+            return false;
+        return Date.now() - this.lastHeartbeatReceivedAt < this.leaderLeaseDurationMs;
+    }
+    /**
+     * The fencing token is the current Raft term. Operations tagged with a
+     * lower term than ours must be rejected to prevent stale-leader writes.
+     */
+    getFencingToken() {
+        return this.currentTerm;
+    }
+    /**
+     * Validates a fencing token from a remote node. Returns true when the
+     * token is at least as recent as our current term.
+     */
+    validateFencingToken(token) {
+        return token >= this.currentTerm;
+    }
     initializeRaftStore(hiveDir) {
         if (this.raftStore)
             return;
@@ -43,6 +88,12 @@ export class RaftStateMachine {
                 return;
             if (this.role === 'leader')
                 return;
+            // Do not start elections while catching up from a snapshot — the node
+            // must not become leader until it has a complete, current state.
+            if (this.isCatchingUp) {
+                this.resetElectionDeadline();
+                return;
+            }
             if (Date.now() >= this.electionDeadline) {
                 void this.startElection().catch(error => this.deps.handleBackgroundError(error));
             }
@@ -72,7 +123,7 @@ export class RaftStateMachine {
         });
         let votes = 1;
         try {
-            await Promise.all(this.config.peers
+            await Promise.all(this.getPeers()
                 .filter(peer => peer.id !== this.config.node_id)
                 .map(async (peer) => {
                 const response = await this.deps.postJson(peer, '/cluster/v1/election/request-vote', {
@@ -143,6 +194,7 @@ export class RaftStateMachine {
         this.role = 'follower';
         this.votedFor = null;
         this.leaderId = leaderId;
+        this.lastHeartbeatReceivedAt = 0;
         this.resetElectionDeadline();
         this.persistRaftState();
         this.appendDurableEntry('state_transition', {
@@ -153,7 +205,7 @@ export class RaftStateMachine {
         });
     }
     quorum() {
-        const nodes = this.config.peers.length + 1;
+        const nodes = this.getPeers().length + 1;
         return Math.floor(nodes / 2) + 1;
     }
     resetElectionDeadline() {
@@ -195,12 +247,22 @@ export class RaftStateMachine {
     getRaftStoreState() {
         return this.raftStore?.getState() ?? null;
     }
+    getLogEntryCount() {
+        return this.raftStore?.getLogEntryCount() ?? 0;
+    }
+    createSnapshotAndCompact(versionVector) {
+        if (!this.raftStore) {
+            return { entries_removed: 0, entries_retained: 0, snapshot_index: 0 };
+        }
+        this.raftStore.createSnapshot(versionVector);
+        return this.raftStore.compactLog();
+    }
     getLeaderUrl() {
         if (!this.leaderId)
             return null;
         if (this.leaderId === this.config.node_id)
             return this.config.public_url;
-        const peer = this.config.peers.find(item => item.id === this.leaderId);
+        const peer = this.getPeers().find(item => item.id === this.leaderId);
         return peer?.url || null;
     }
 }