diskeyval 1.0.0

package/test/e2e/cluster.e2e.test.ts

@@ -0,0 +1,280 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import diskeyval, { type DiskeyvalNode } from '../../lib/index.ts';
+import { isNotLeaderError, isQuorumUnavailableError } from '../../lib/index.ts';
+import { hasOpenSsl } from '../support/certs.ts';
+import { createTestCertificates } from '../support/certs.ts';
+import { getFreePorts } from '../support/ports.ts';
+import { createTlsTestCluster } from '../support/tls-cluster.ts';
+import { waitForCondition } from '../support/helpers.ts';
+
+async function forceLeader(node: { forceElection: () => Promise<void>; isLeader: () => boolean }) {
+  await node.forceElection();
+  await waitForCondition(() => node.isLeader(), {
+    timeoutMs: 1_500,
+    intervalMs: 20,
+    message: 'Expected forced node to become leader'
+  });
+}
+
+test('mTLS cluster elects leader and replicates writes', { skip: !hasOpenSsl() }, async () => {
+  const cluster = await createTlsTestCluster(['node-1', 'node-2', 'node-3']);
+
+  try {
+    await cluster.start();
+    await forceLeader(cluster.nodes['node-1']);
+
+    await cluster.nodes['node-1'].set('e2e-key', 'e2e-value');
+
+    await waitForCondition(
+      () =>
+        cluster.nodes['node-1'].state['e2e-key'] === 'e2e-value' &&
+        cluster.nodes['node-2'].state['e2e-key'] === 'e2e-value' &&
+        cluster.nodes['node-3'].state['e2e-key'] === 'e2e-value',
+      {
+        timeoutMs: 3_000,
+        intervalMs: 25,
+        message: 'mTLS cluster did not converge on committed value'
+      }
+    );
+  } finally {
+    await cluster.stop();
+    cluster.cleanup();
+  }
+});
+
+test('mTLS cluster rejects writes when quorum is unavailable', { skip: !hasOpenSsl() }, async () => {
+  const cluster = await createTlsTestCluster(['node-1', 'node-2', 'node-3']);
+
+  try {
+    await cluster.start();
+    await forceLeader(cluster.nodes['node-1']);
+
+    await cluster.nodes['node-2'].end();
+    await cluster.nodes['node-3'].end();
+
+    await assert.rejects(
+      () => cluster.nodes['node-1'].set('blocked-write', true),
+      (error: unknown) => isQuorumUnavailableError(error)
+    );
+  } finally {
+    await cluster.stop();
+    cluster.cleanup();
+  }
+});
+
+test('mTLS cluster performs failover when leader stops', { skip: !hasOpenSsl() }, async () => {
+  const cluster = await createTlsTestCluster(['node-1', 'node-2', 'node-3']);
+
+  try {
+    await cluster.start();
+    await forceLeader(cluster.nodes['node-1']);
+
+    await cluster.nodes['node-1'].end();
+
+    await waitForCondition(
+      () => cluster.nodes['node-2'].isLeader() || cluster.nodes['node-3'].isLeader(),
+      {
+        timeoutMs: 4_000,
+        intervalMs: 25,
+        message: 'No new leader elected after stopping previous leader'
+      }
+    );
+
+    const newLeader = cluster.nodes['node-2'].isLeader() ? cluster.nodes['node-2'] : cluster.nodes['node-3'];
+    await newLeader.set('post-failover', 'ok');
+
+    await waitForCondition(
+      () => cluster.nodes['node-2'].state['post-failover'] === 'ok' || cluster.nodes['node-3'].state['post-failover'] === 'ok',
+      {
+        timeoutMs: 3_000,
+        intervalMs: 25,
+        message: 'New leader did not replicate post-failover write'
+      }
+    );
+  } finally {
+    await cluster.stop();
+    cluster.cleanup();
+  }
+});
+
+test('follower mTLS read returns not-leader error with leader hint', { skip: !hasOpenSsl() }, async () => {
+  const cluster = await createTlsTestCluster(['node-1', 'node-2', 'node-3']);
+
+  try {
+    await cluster.start();
+    await forceLeader(cluster.nodes['node-1']);
+    await waitForCondition(() => cluster.nodes['node-2'].leaderId() === 'node-1', {
+      timeoutMs: 1_500,
+      intervalMs: 20,
+      message: 'Follower did not observe leader identity'
+    });
+
+    await assert.rejects(
+      () => cluster.nodes['node-2'].get('missing'),
+      (error: unknown) => {
+        assert.ok(isNotLeaderError(error));
+        assert.equal(error.leaderId, 'node-1');
+        return true;
+      }
+    );
+  } finally {
+    await cluster.stop();
+    cluster.cleanup();
+  }
+});
+
+test('mTLS cluster restores committed state after full restart', { skip: !hasOpenSsl() }, async () => {
+  const persistenceDir = await mkdtemp(join(tmpdir(), 'diskeyval-e2e-persist-'));
+
+  const firstRun = await createTlsTestCluster(['node-1', 'node-2', 'node-3'], {
+    persistenceDir
+  });
+  let committedWriterId: 'node-1' | 'node-2' | 'node-3' = 'node-1';
+
+  const getLeader = (nodes: typeof firstRun.nodes) => {
+    if (nodes['node-1'].isLeader()) {
+      return nodes['node-1'];
+    }
+    if (nodes['node-2'].isLeader()) {
+      return nodes['node-2'];
+    }
+    if (nodes['node-3'].isLeader()) {
+      return nodes['node-3'];
+    }
+    return null;
+  };
+
+  try {
+    await firstRun.start();
+    await waitForCondition(() => getLeader(firstRun.nodes) !== null, {
+      timeoutMs: 3_000,
+      intervalMs: 20,
+      message: 'Expected a leader before initial persistent write'
+    });
+
+    const initialLeader = getLeader(firstRun.nodes);
+    assert.ok(initialLeader);
+    await initialLeader.set('restart-key', 'restart-value');
+    committedWriterId = initialLeader.nodeId as 'node-1' | 'node-2' | 'node-3';
+  } finally {
+    await firstRun.stop();
+    firstRun.cleanup();
+  }
+
+  const secondRun = await createTlsTestCluster(['node-1', 'node-2', 'node-3'], {
+    persistenceDir
+  });
+
+  try {
+    await secondRun.start();
+    await waitForCondition(
+      () => secondRun.nodes[committedWriterId].state['restart-key'] === 'restart-value',
+      {
+        timeoutMs: 8_000,
+        intervalMs: 25,
+        message: 'Expected persisted state to be restored on restart for committed writer'
+      }
+    );
+
+    await forceLeader(secondRun.nodes[committedWriterId]);
+    const recovered = await secondRun.nodes[committedWriterId].get('restart-key');
+    assert.equal(recovered, 'restart-value');
+  } finally {
+    await secondRun.stop();
+    secondRun.cleanup();
+    await rm(persistenceDir, { recursive: true, force: true });
+  }
+});
+
+test('mTLS cluster can dynamically add a new node and replicate state', { skip: !hasOpenSsl() }, async () => {
+  const nodeIds = ['node-1', 'node-2', 'node-3', 'node-4'];
+  const certs = createTestCertificates(nodeIds);
+  const ports = await getFreePorts(nodeIds.length);
+  const endpoints = Object.fromEntries(
+    nodeIds.map((nodeId, index) => [
+      nodeId,
+      {
+        nodeId,
+        host: '127.0.0.1',
+        port: ports[index]
+      }
+    ])
+  ) as Record<string, { nodeId: string; host: string; port: number }>;
+
+  const mkNode = (nodeId: string, peers: string[]): DiskeyvalNode =>
+    diskeyval({
+      nodeId,
+      host: endpoints[nodeId].host,
+      port: endpoints[nodeId].port,
+      peers: peers.map((peerId) => ({
+        nodeId: endpoints[peerId].nodeId,
+        host: endpoints[peerId].host,
+        port: endpoints[peerId].port
+      })),
+      tls: {
+        cert: certs.nodes[nodeId].certPem,
+        key: certs.nodes[nodeId].keyPem,
+        ca: certs.caPem
+      },
+      electionTimeoutMs: 200,
+      heartbeatMs: 50,
+      proposalTimeoutMs: 2_000,
+      rpcTimeoutMs: 2_000
+    });
+
+  const node1 = mkNode('node-1', ['node-2', 'node-3']);
+  const node2 = mkNode('node-2', ['node-1', 'node-3']);
+  const node3 = mkNode('node-3', ['node-1', 'node-2']);
+  const node4 = mkNode('node-4', ['node-2']);
+
+  try {
+    await node1.start();
+    await node2.start();
+    await node3.start();
+
+    await forceLeader(node1);
+    await node1.set('before-join', 'present');
+
+    await node4.start();
+
+    await node1.reconfigure([
+      endpoints['node-2'],
+      endpoints['node-3'],
+      endpoints['node-4']
+    ]);
+
+    await waitForCondition(
+      () => node4.state['before-join'] === 'present',
+      {
+        timeoutMs: 5_000,
+        intervalMs: 30,
+        message: 'new node did not catch up existing committed state after reconfigure'
+      }
+    );
+
+    await node1.set('after-join', 'replicated');
+    await waitForCondition(
+      () =>
+        node2.state['after-join'] === 'replicated' &&
+        node3.state['after-join'] === 'replicated' &&
+        node4.state['after-join'] === 'replicated',
+      {
+        timeoutMs: 5_000,
+        intervalMs: 30,
+        message: 'new cluster membership did not replicate subsequent writes'
+      }
+    );
+  } finally {
+    await Promise.all([
+      node1.end().catch(() => undefined),
+      node2.end().catch(() => undefined),
+      node3.end().catch(() => undefined),
+      node4.end().catch(() => undefined)
+    ]);
+    certs.cleanup();
+  }
+});

package/test/sim/cluster.sim.test.ts

@@ -0,0 +1,105 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { isQuorumUnavailableError } from '../../lib/index.ts';
+import { createTestCluster, forceLeader, waitForCondition } from '../support/helpers.ts';
+
+test('[sim] 3-node cluster elects leader and commits value', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+    await cluster.nodes['node-1'].set('sim-key', 'sim-value');
+
+    await waitForCondition(
+      () =>
+        cluster.nodes['node-1'].state['sim-key'] === 'sim-value' &&
+        cluster.nodes['node-2'].state['sim-key'] === 'sim-value' &&
+        cluster.nodes['node-3'].state['sim-key'] === 'sim-value',
+      { timeoutMs: 1_000, message: 'Cluster did not converge' }
+    );
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('[sim] minority partition cannot commit', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+    cluster.network.partition('node-1', 'node-2');
+    cluster.network.partition('node-1', 'node-3');
+
+    await assert.rejects(
+      () => cluster.nodes['node-1'].set('blocked', 1),
+      (error: unknown) => isQuorumUnavailableError(error)
+    );
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('[sim] majority partition continues committing', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    cluster.network.partition('node-1', 'node-3');
+    cluster.network.partition('node-2', 'node-3');
+
+    await cluster.nodes['node-1'].set('majority', true);
+    await waitForCondition(
+      () => cluster.nodes['node-2'].state.majority === true,
+      { timeoutMs: 500, message: 'Majority peer did not commit write' }
+    );
+
+    assert.equal(cluster.nodes['node-3'].state.majority, undefined);
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('[sim] partition heal converges logs', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    cluster.network.partition('node-1', 'node-3');
+    cluster.network.partition('node-2', 'node-3');
+    await cluster.nodes['node-1'].set('heal-key', 'during-partition');
+
+    cluster.network.healAll();
+
+    await waitForCondition(
+      () => cluster.nodes['node-3'].state['heal-key'] === 'during-partition',
+      { timeoutMs: 1_200, message: 'Lagging node did not catch up after heal' }
+    );
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('[sim] dynamic reconfigure can add a new node and replicate state', async () => {
+  const cluster = createTestCluster(['node-1', 'node-2', 'node-3', 'node-4']);
+  await cluster.start();
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    await cluster.nodes['node-1'].reconfigure([
+      { nodeId: 'node-2' },
+      { nodeId: 'node-3' },
+      { nodeId: 'node-4' }
+    ]);
+
+    await cluster.nodes['node-1'].set('new-member-key', 'replicated');
+
+    await waitForCondition(
+      () => cluster.nodes['node-4'].state['new-member-key'] === 'replicated',
+      { timeoutMs: 1_500, message: 'new member did not catch up after reconfigure' }
+    );
+  } finally {
+    await cluster.stop();
+  }
+});

package/test/soak/cluster.soak.test.ts

@@ -0,0 +1,97 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createTestCluster, forceLeader, waitForCondition } from '../support/helpers.ts';
+
+test('high write throughput burst converges on all replicas', async () => {
+  const cluster = createTestCluster(['node-1', 'node-2', 'node-3', 'node-4', 'node-5']);
+  await cluster.start();
+
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    const writeCount = 120;
+    for (let index = 0; index < writeCount; index += 1) {
+      await cluster.nodes['node-1'].set(`k-${index}`, index);
+    }
+
+    await waitForCondition(
+      () => cluster.nodes['node-4'].state['k-119'] === 119,
+      {
+        timeoutMs: 2_000,
+        intervalMs: 15,
+        message: 'Tail write did not replicate in throughput burst'
+      }
+    );
+
+    assert.equal(cluster.nodes['node-1'].state['k-0'], 0);
+    assert.equal(cluster.nodes['node-2'].state['k-50'], 50);
+    assert.equal(cluster.nodes['node-3'].state['k-119'], 119);
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('cluster recovers from repeated partition/heal cycles', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    for (let cycle = 0; cycle < 8; cycle += 1) {
+      cluster.network.partition('node-1', 'node-3');
+      cluster.network.partition('node-2', 'node-3');
+      await cluster.nodes['node-1'].set(`cycle-${cycle}`, cycle);
+      cluster.network.healAll();
+
+      await waitForCondition(
+        () => cluster.nodes['node-3'].state[`cycle-${cycle}`] === cycle,
+        {
+          timeoutMs: 1_500,
+          intervalMs: 20,
+          message: `Lagging follower did not catch up after cycle ${cycle}`
+        }
+      );
+    }
+  } finally {
+    await cluster.stop();
+  }
+});
+
+test('randomized partition churn preserves convergence safety', async () => {
+  const cluster = createTestCluster();
+  await cluster.start();
+
+  try {
+    await forceLeader(cluster.nodes['node-1']);
+
+    for (let index = 0; index < 50; index += 1) {
+      if (index % 7 === 0) {
+        cluster.network.partition('node-1', 'node-3');
+        cluster.network.partition('node-2', 'node-3');
+      }
+      if (index % 7 === 3) {
+        cluster.network.healAll();
+      }
+
+      await cluster.nodes['node-1'].set(`safe-${index}`, index);
+    }
+
+    cluster.network.healAll();
+
+    await waitForCondition(
+      () => cluster.nodes['node-3'].state['safe-49'] === 49,
+      {
+        timeoutMs: 2_000,
+        intervalMs: 20,
+        message: 'Cluster failed to converge after randomized churn'
+      }
+    );
+
+    assert.equal(cluster.nodes['node-1'].state['safe-49'], 49);
+    assert.equal(cluster.nodes['node-2'].state['safe-49'], 49);
+    assert.equal(cluster.nodes['node-3'].state['safe-49'], 49);
+  } finally {
+    await cluster.stop();
+  }
+});

package/test/support/certs.ts

@@ -0,0 +1,126 @@
+import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { execFileSync, spawnSync } from 'node:child_process';
+
+function runOpenSsl(args: string[], cwd: string): void {
+  execFileSync('openssl', args, {
+    cwd,
+    stdio: 'pipe'
+  });
+}
+
+export function hasOpenSsl(): boolean {
+  const result = spawnSync('openssl', ['version'], { stdio: 'pipe' });
+  return result.status === 0;
+}
+
+export type NodeCertificateBundle = {
+  nodeId: string;
+  certPem: string;
+  keyPem: string;
+};
+
+export type TestCertificates = {
+  caPem: string;
+  nodes: Record<string, NodeCertificateBundle>;
+  cleanup: () => void;
+};
+
+export function createTestCertificates(nodeIds: string[]): TestCertificates {
+  const workingDir = mkdtempSync(join(tmpdir(), 'diskeyval-certs-'));
+
+  const caKeyPath = join(workingDir, 'ca.key.pem');
+  const caCertPath = join(workingDir, 'ca.cert.pem');
+
+  runOpenSsl(['genrsa', '-out', caKeyPath, '2048'], workingDir);
+  runOpenSsl(
+    [
+      'req',
+      '-x509',
+      '-new',
+      '-nodes',
+      '-key',
+      caKeyPath,
+      '-sha256',
+      '-days',
+      '1',
+      '-out',
+      caCertPath,
+      '-subj',
+      '/CN=diskeyval-test-ca'
+    ],
+    workingDir
+  );
+
+  const caPem = readFileSync(caCertPath, 'utf8');
+  const nodes: Record<string, NodeCertificateBundle> = {};
+
+  for (const nodeId of nodeIds) {
+    const keyPath = join(workingDir, `${nodeId}.key.pem`);
+    const csrPath = join(workingDir, `${nodeId}.csr.pem`);
+    const certPath = join(workingDir, `${nodeId}.cert.pem`);
+    const configPath = join(workingDir, `${nodeId}.cnf`);
+
+    writeFileSync(
+      configPath,
+      [
+        '[req]',
+        'prompt = no',
+        'distinguished_name = dn',
+        'req_extensions = req_ext',
+        '',
+        '[dn]',
+        `CN = ${nodeId}`,
+        '',
+        '[req_ext]',
+        `subjectAltName = DNS:${nodeId},URI:spiffe://diskeyval/${nodeId}`,
+        'extendedKeyUsage = serverAuth,clientAuth',
+        'keyUsage = digitalSignature,keyEncipherment'
+      ].join('\n')
+    );
+
+    runOpenSsl(['genrsa', '-out', keyPath, '2048'], workingDir);
+    runOpenSsl(
+      ['req', '-new', '-key', keyPath, '-out', csrPath, '-config', configPath],
+      workingDir
+    );
+    runOpenSsl(
+      [
+        'x509',
+        '-req',
+        '-in',
+        csrPath,
+        '-CA',
+        caCertPath,
+        '-CAkey',
+        caKeyPath,
+        '-CAcreateserial',
+        '-out',
+        certPath,
+        '-days',
+        '1',
+        '-sha256',
+        '-extfile',
+        configPath,
+        '-extensions',
+        'req_ext'
+      ],
+      workingDir
+    );
+
+    nodes[nodeId] = {
+      nodeId,
+      certPem: readFileSync(certPath, 'utf8'),
+      keyPem: readFileSync(keyPath, 'utf8')
+    };
+  }
+
+  return {
+    caPem,
+    nodes,
+    cleanup: () => {
+      rmSync(workingDir, { recursive: true, force: true });
+    }
+  };
+}

package/test/support/helpers.ts

@@ -0,0 +1,51 @@
+import assert from 'node:assert/strict';
+import { createInMemoryDiskeyvalCluster, type DiskeyvalNode } from '../../lib/index.ts';
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitForCondition(
+  condition: () => boolean,
+  options?: { timeoutMs?: number; intervalMs?: number; message?: string }
+): Promise<void> {
+  const timeoutMs = options?.timeoutMs ?? 1_000;
+  const intervalMs = options?.intervalMs ?? 10;
+  const start = Date.now();
+
+  while (Date.now() - start <= timeoutMs) {
+    if (condition()) {
+      return;
+    }
+    await sleep(intervalMs);
+  }
+
+  assert.fail(options?.message ?? `Condition not met within ${timeoutMs}ms`);
+}
+
+export async function forceLeader(node: DiskeyvalNode): Promise<void> {
+  await node.forceElection();
+  await waitForCondition(() => node.isLeader(), {
+    timeoutMs: 500,
+    message: `Node ${node.nodeId} did not become leader`
+  });
+}
+
+export function createTestCluster(nodeIds: string[] = ['node-1', 'node-2', 'node-3']) {
+  return createInMemoryDiskeyvalCluster(nodeIds, {
+    electionTimeoutMs: 10_000,
+    heartbeatMs: 20,
+    proposalTimeoutMs: 200
+  });
+}
+
+export function createTestClusterWithOptions(
+  nodeIds: string[],
+  options: {
+    electionTimeoutMs?: number;
+    heartbeatMs?: number;
+    proposalTimeoutMs?: number;
+  }
+) {
+  return createInMemoryDiskeyvalCluster(nodeIds, options);
+}

package/test/support/ports.ts

@@ -0,0 +1,33 @@
+import { createServer } from 'node:net';
+
+export async function getFreePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+
+    server.once('error', reject);
+    server.listen(0, '127.0.0.1', () => {
+      const address = server.address();
+      if (!address || typeof address === 'string') {
+        server.close(() => reject(new Error('Could not allocate free port')));
+        return;
+      }
+
+      const port = address.port;
+      server.close((closeError) => {
+        if (closeError) {
+          reject(closeError);
+          return;
+        }
+        resolve(port);
+      });
+    });
+  });
+}
+
+export async function getFreePorts(count: number): Promise<number[]> {
+  const ports: number[] = [];
+  for (let index = 0; index < count; index += 1) {
+    ports.push(await getFreePort());
+  }
+  return ports;
+}