nodio-cli 1.0.0

package/src/server/routes.js

@@ -0,0 +1,372 @@
+const express = require('express');
+const { v4: uuidv4 } = require('uuid');
+const {
+  NodeModel,
+  FileModel,
+  ShardModel,
+  ShardPlacementModel,
+  ReplicationTaskModel
+} = require('./models');
+const {
+  chooseDistinctOnlineNodes,
+  ensureEmergencyReplicasForShard
+} = require('./services');
+
+function parsePositiveInt(value, fallback) {
+  const n = Number(value);
+  return Number.isInteger(n) && n > 0 ? n : fallback;
+}
+
+function buildRoutes(config) {
+  const router = express.Router();
+
+  router.get('/health', async (_req, res) => {
+    res.json({ ok: true, service: 'nodio-server' });
+  });
+
+  router.post('/nodes/register', async (req, res, next) => {
+    try {
+      const { nodeId, url, capacityBytes, freeBytes } = req.body;
+
+      if (!nodeId || !url) {
+        return res.status(400).json({ error: 'nodeId and url are required' });
+      }
+
+      const capacity = Number(capacityBytes);
+      const free = Number(freeBytes);
+      if (!Number.isFinite(capacity) || capacity <= 0 || !Number.isFinite(free) || free < 0) {
+        return res.status(400).json({ error: 'capacityBytes and freeBytes must be valid numbers' });
+      }
+
+      const node = await NodeModel.findOneAndUpdate(
+        { nodeId },
+        {
+          $set: {
+            url,
+            capacityBytes: capacity,
+            freeBytes: free,
+            status: 'online',
+            lastHeartbeatAt: new Date()
+          }
+        },
+        { upsert: true, new: true, setDefaultsOnInsert: true }
+      );
+
+      res.json({
+        nodeId: node.nodeId,
+        status: node.status,
+        heartbeatIntervalMs: config.heartbeatIntervalMs,
+        minReplicas: config.minReplicas,
+        emergencyReplicaFloor: config.emergencyReplicaFloor
+      });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  router.post('/nodes/heartbeat', async (req, res, next) => {
+    try {
+      const { nodeId, freeBytes, shardIds } = req.body;
+      if (!nodeId) {
+        return res.status(400).json({ error: 'nodeId is required' });
+      }
+
+      const node = await NodeModel.findOne({ nodeId });
+      if (!node) {
+        return res.status(404).json({ error: 'node not found' });
+      }
+
+      node.status = 'online';
+      node.lastHeartbeatAt = new Date();
+      if (Number.isFinite(Number(freeBytes)) && Number(freeBytes) >= 0) {
+        node.freeBytes = Number(freeBytes);
+      }
+      await node.save();
+
+      if (Array.isArray(shardIds) && shardIds.length > 0) {
+        const normalizedShardIds = [...new Set(shardIds.filter(Boolean))];
+        const knownShards = await ShardModel.find({ shardId: { $in: normalizedShardIds } })
+          .select('shardId fileId')
+          .lean();
+
+        for (const shard of knownShards) {
+          await ShardPlacementModel.updateOne(
+            { shardId: shard.shardId, nodeId },
+            {
+              $set: {
+                status: 'available',
+                fileId: shard.fileId,
+                updatedAt: new Date()
+              },
+              $setOnInsert: {
+                shardId: shard.shardId,
+                nodeId,
+                fileId: shard.fileId,
+                createdAt: new Date()
+              }
+            },
+            { upsert: true }
+          );
+        }
+      }
+
+      const pendingTasks = await ReplicationTaskModel.find({
+        targetNodeId: nodeId,
+        status: 'pending'
+      })
+        .sort({ createdAt: 1 })
+        .limit(5)
+        .lean();
+
+      if (pendingTasks.length > 0) {
+        await ReplicationTaskModel.updateMany(
+          { _id: { $in: pendingTasks.map((task) => task._id) } },
+          { $set: { status: 'in_progress' }, $inc: { attempts: 1 } }
+        );
+      }
+
+      const tasksWithSourceUrl = [];
+      for (const task of pendingTasks) {
+        const source = await NodeModel.findOne({ nodeId: task.sourceNodeId, status: 'online' })
+          .select('url nodeId')
+          .lean();
+
+        if (!source) {
+          await ReplicationTaskModel.updateOne(
+            { _id: task._id },
+            {
+              $set: {
+                status: 'failed',
+                errorMessage: 'source node is offline'
+              }
+            }
+          );
+          continue;
+        }
+
+        tasksWithSourceUrl.push({
+          taskId: task._id.toString(),
+          shardId: task.shardId,
+          fileId: task.fileId,
+          sourceNodeId: source.nodeId,
+          sourceUrl: source.url
+        });
+      }
+
+      res.json({
+        ok: true,
+        now: new Date().toISOString(),
+        replicationTasks: tasksWithSourceUrl
+      });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  router.post('/replication-tasks/:taskId/complete', async (req, res, next) => {
+    try {
+      const { taskId } = req.params;
+      const { nodeId, success, errorMessage } = req.body;
+
+      const task = await ReplicationTaskModel.findById(taskId);
+      if (!task) {
+        return res.status(404).json({ error: 'replication task not found' });
+      }
+
+      if (task.targetNodeId !== nodeId) {
+        return res.status(403).json({ error: 'nodeId does not match task target' });
+      }
+
+      if (success) {
+        task.status = 'completed';
+        task.errorMessage = null;
+        await task.save();
+
+        await ShardPlacementModel.updateOne(
+          { shardId: task.shardId, nodeId: task.targetNodeId },
+          {
+            $set: {
+              fileId: task.fileId,
+              status: 'available'
+            },
+            $setOnInsert: {
+              shardId: task.shardId,
+              nodeId: task.targetNodeId
+            }
+          },
+          { upsert: true }
+        );
+
+        await ensureEmergencyReplicasForShard(task.shardId, config.emergencyReplicaFloor);
+      } else {
+        task.status = 'failed';
+        task.errorMessage = errorMessage || 'replication failed';
+        await task.save();
+      }
+
+      res.json({ ok: true });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  router.post('/files/register', async (req, res, next) => {
+    try {
+      const { fileId, originalName, sizeBytes, shardCount, cipher, metadata } = req.body;
+      if (!originalName || !Number.isFinite(Number(sizeBytes)) || Number(sizeBytes) < 0) {
+        return res.status(400).json({ error: 'originalName and sizeBytes are required' });
+      }
+
+      const normalizedShardCount = parsePositiveInt(shardCount, 1);
+      const actualFileId = fileId || uuidv4();
+
+      const file = await FileModel.findOneAndUpdate(
+        { fileId: actualFileId },
+        {
+          $set: {
+            fileId: actualFileId,
+            originalName,
+            sizeBytes: Number(sizeBytes),
+            shardCount: normalizedShardCount,
+            cipher: cipher || 'aes-256-gcm',
+            metadata: metadata || {}
+          }
+        },
+        { upsert: true, new: true, setDefaultsOnInsert: true }
+      );
+
+      res.json({ fileId: file.fileId, shardCount: file.shardCount });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  router.post('/shards/register', async (req, res, next) => {
+    try {
+      const { shardId, fileId, order, sizeBytes, checksum, nodeIds } = req.body;
+      if (!shardId || !fileId || !checksum || !Array.isArray(nodeIds) || nodeIds.length === 0) {
+        return res.status(400).json({ error: 'shardId, fileId, checksum and nodeIds are required' });
+      }
+
+      await ShardModel.findOneAndUpdate(
+        { shardId },
+        {
+          $set: {
+            shardId,
+            fileId,
+            order: Number(order) || 0,
+            sizeBytes: Number(sizeBytes) || 0,
+            checksum
+          }
+        },
+        { upsert: true, new: true, setDefaultsOnInsert: true }
+      );
+
+      const uniqueNodeIds = [...new Set(nodeIds)];
+      for (const nodeId of uniqueNodeIds) {
+        await ShardPlacementModel.updateOne(
+          { shardId, nodeId },
+          {
+            $set: {
+              fileId,
+              status: 'available'
+            },
+            $setOnInsert: {
+              shardId,
+              nodeId
+            }
+          },
+          { upsert: true }
+        );
+      }
+
+      await ensureEmergencyReplicasForShard(shardId, config.emergencyReplicaFloor);
+
+      res.json({ ok: true });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  router.post('/shards/placement-plan', async (req, res, next) => {
+    try {
+      const { shardId, sizeBytes, replicas } = req.body;
+      if (!shardId || !Number.isFinite(Number(sizeBytes)) || Number(sizeBytes) < 0) {
+        return res.status(400).json({ error: 'shardId and sizeBytes are required' });
+      }
+
+      const replicaCount = parsePositiveInt(replicas, config.minReplicas);
+      const nodes = await chooseDistinctOnlineNodes(replicaCount, Number(sizeBytes));
+
+      res.json({
+        shardId,
+        replicas: nodes.map((node) => ({
+          nodeId: node.nodeId,
+          url: node.url
+        }))
+      });
+    } catch (error) {
+      if (String(error.message).startsWith('insufficient_online_nodes')) {
+        return res.status(409).json({ error: error.message });
+      }
+      next(error);
+    }
+  });
+
+  router.get('/files/:fileId/manifest', async (req, res, next) => {
+    try {
+      const { fileId } = req.params;
+      const file = await FileModel.findOne({ fileId }).lean();
+      if (!file) {
+        return res.status(404).json({ error: 'file not found' });
+      }
+
+      const shards = await ShardModel.find({ fileId }).sort({ order: 1 }).lean();
+      const placements = await ShardPlacementModel.find({ fileId, status: 'available' }).lean();
+
+      const nodeIds = [...new Set(placements.map((placement) => placement.nodeId))];
+      const nodes = await NodeModel.find({ nodeId: { $in: nodeIds }, status: 'online' })
+        .select('nodeId url')
+        .lean();
+      const nodeMap = new Map(nodes.map((node) => [node.nodeId, node.url]));
+
+      const shardManifests = shards.map((shard) => {
+        const shardPlacements = placements
+          .filter((placement) => placement.shardId === shard.shardId)
+          .map((placement) => ({
+            nodeId: placement.nodeId,
+            url: nodeMap.get(placement.nodeId)
+          }))
+          .filter((placement) => Boolean(placement.url));
+
+        return {
+          shardId: shard.shardId,
+          order: shard.order,
+          sizeBytes: shard.sizeBytes,
+          checksum: shard.checksum,
+          replicas: shardPlacements
+        };
+      });
+
+      res.json({
+        file: {
+          fileId: file.fileId,
+          originalName: file.originalName,
+          sizeBytes: file.sizeBytes,
+          shardCount: file.shardCount,
+          cipher: file.cipher,
+          metadata: file.metadata
+        },
+        shards: shardManifests
+      });
+    } catch (error) {
+      next(error);
+    }
+  });
+
+  return router;
+}
+
+module.exports = {
+  buildRoutes
+};

package/src/server/services.js

@@ -0,0 +1,109 @@
+const {
+  NodeModel,
+  ShardModel,
+  ShardPlacementModel,
+  ReplicationTaskModel
+} = require('./models');
+
+async function chooseDistinctOnlineNodes(requiredCount, minFreeBytes = 0, excludedNodeIds = []) {
+  const excluded = new Set(excludedNodeIds);
+
+  const candidates = await NodeModel.find({
+    status: 'online',
+    freeBytes: { $gte: minFreeBytes },
+    nodeId: { $nin: [...excluded] }
+  })
+    .sort({ freeBytes: -1, lastHeartbeatAt: -1 })
+    .lean();
+
+  if (candidates.length < requiredCount) {
+    throw new Error(`insufficient_online_nodes: required ${requiredCount}, found ${candidates.length}`);
+  }
+
+  return candidates.slice(0, requiredCount);
+}
+
+async function ensureEmergencyReplicasForShard(shardId, emergencyReplicaFloor) {
+  const shard = await ShardModel.findOne({ shardId }).lean();
+  if (!shard) {
+    return;
+  }
+
+  const existingPlacements = await ShardPlacementModel.find({
+    shardId,
+    status: 'available'
+  }).lean();
+
+  const onlineMap = new Map();
+  const onlineNodes = await NodeModel.find({ status: 'online' }).lean();
+  for (const node of onlineNodes) {
+    onlineMap.set(node.nodeId, node);
+  }
+
+  const livePlacements = existingPlacements.filter((p) => onlineMap.has(p.nodeId));
+  const currentReplicaCount = livePlacements.length;
+
+  if (currentReplicaCount >= emergencyReplicaFloor) {
+    return;
+  }
+
+  const sourcePlacement = livePlacements[0];
+  if (!sourcePlacement) {
+    return;
+  }
+
+  const missing = emergencyReplicaFloor - currentReplicaCount;
+  const excluded = livePlacements.map((p) => p.nodeId);
+  const targets = await chooseDistinctOnlineNodes(missing, shard.sizeBytes, excluded);
+
+  for (const target of targets) {
+    await ReplicationTaskModel.updateOne(
+      {
+        shardId,
+        targetNodeId: target.nodeId,
+        status: { $in: ['pending', 'in_progress'] }
+      },
+      {
+        $setOnInsert: {
+          shardId,
+          fileId: shard.fileId,
+          sourceNodeId: sourcePlacement.nodeId,
+          targetNodeId: target.nodeId,
+          status: 'pending',
+          attempts: 0
+        }
+      },
+      { upsert: true }
+    );
+  }
+}
+
+async function markNodeOfflineAndRecover(nodeId, emergencyReplicaFloor) {
+  const node = await NodeModel.findOne({ nodeId });
+  if (!node || node.status === 'offline') {
+    return false;
+  }
+
+  node.status = 'offline';
+  await node.save();
+
+  await ShardPlacementModel.updateMany(
+    { nodeId, status: 'available' },
+    { $set: { status: 'lost' } }
+  );
+
+  const affectedShardRows = await ShardPlacementModel.find({ nodeId }).select('shardId').lean();
+  const shardIds = [...new Set(affectedShardRows.map((row) => row.shardId))];
+
+  for (const shardId of shardIds) {
+    await ensureEmergencyReplicasForShard(shardId, emergencyReplicaFloor);
+  }
+
+  return true;
+}
+
+module.exports = {
+  chooseDistinctOnlineNodes,
+  ensureEmergencyReplicasForShard,
+  markNodeOfflineAndRecover
+};

package/src/user/client.js

@@ -0,0 +1,20 @@
+const axios = require('axios');
+
+function normalizeUrl(url) {
+  return String(url || '').replace(/\/+$/, '');
+}
+
+function createApiClient(serverUrl) {
+  return axios.create({
+    baseURL: `${normalizeUrl(serverUrl)}/api`,
+    timeout: 30000,
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  });
+}
+
+module.exports = {
+  createApiClient,
+  normalizeUrl
+};

package/src/user/commands.js

@@ -0,0 +1,205 @@
+const fs = require('fs/promises');
+const path = require('path');
+const crypto = require('crypto');
+const axios = require('axios');
+const { v4: uuidv4 } = require('uuid');
+const { createApiClient, normalizeUrl } = require('./client');
+const { encryptAes256Gcm, decryptAes256Gcm, sha256Hex } = require('../common/crypto');
+
+function splitBuffer(buffer, shardSizeBytes) {
+  if (shardSizeBytes <= 0) {
+    throw new Error('shard size must be greater than zero');
+  }
+
+  const chunks = [];
+  for (let offset = 0; offset < buffer.length; offset += shardSizeBytes) {
+    chunks.push(buffer.subarray(offset, Math.min(offset + shardSizeBytes, buffer.length)));
+  }
+
+  return chunks.length > 0 ? chunks : [Buffer.alloc(0)];
+}
+
+function parseAesKey(keyBase64) {
+  const key = Buffer.from(keyBase64, 'base64');
+  if (key.length !== 32) {
+    throw new Error('key must decode to exactly 32 bytes for AES-256');
+  }
+  return key;
+}
+
+async function uploadFile(options) {
+  const filePath = path.resolve(options.file);
+  const serverUrl = options.server;
+  const shardSizeMb = Number(options.shardSizeMb || 1);
+  const replicas = Number(options.replicas || 5);
+
+  if (!Number.isFinite(shardSizeMb) || shardSizeMb <= 0) {
+    throw new Error('shard-size-mb must be greater than 0');
+  }
+  if (!Number.isInteger(replicas) || replicas < 5) {
+    throw new Error('replicas must be an integer >= 5');
+  }
+
+  const shardSizeBytes = Math.floor(shardSizeMb * 1024 * 1024);
+  const plainBuffer = await fs.readFile(filePath);
+  const chunks = splitBuffer(plainBuffer, shardSizeBytes);
+
+  const keyBuffer = options.keyBase64 ? parseAesKey(options.keyBase64) : crypto.randomBytes(32);
+  const keyBase64 = keyBuffer.toString('base64');
+
+  const api = createApiClient(serverUrl);
+  const fileId = options.fileId || uuidv4();
+  const originalName = path.basename(filePath);
+
+  await api.post('/files/register', {
+    fileId,
+    originalName,
+    sizeBytes: plainBuffer.length,
+    shardCount: chunks.length,
+    cipher: 'aes-256-gcm',
+    metadata: {
+      encryption: {
+        algorithm: 'aes-256-gcm',
+        shardSizeBytes,
+        keyFormat: 'base64',
+        shards: []
+      }
+    }
+  });
+
+  const encryptionShardMeta = [];
+
+  for (let order = 0; order < chunks.length; order += 1) {
+    const shardId = `${fileId}-shard-${order}-${uuidv4().slice(0, 8)}`;
+    const encrypted = encryptAes256Gcm(chunks[order], keyBuffer);
+    const checksum = sha256Hex(encrypted.cipherText);
+
+    const placementResponse = await api.post('/shards/placement-plan', {
+      shardId,
+      sizeBytes: encrypted.cipherText.length,
+      replicas
+    });
+
+    const plannedReplicas = placementResponse.data.replicas || [];
+    if (plannedReplicas.length < replicas) {
+      throw new Error(`placement failed for ${shardId}: expected ${replicas} replicas`);
+    }
+
+    for (const replica of plannedReplicas) {
+      const putUrl = `${normalizeUrl(replica.url)}/shards/${shardId}`;
+      await axios.put(putUrl, encrypted.cipherText, {
+        headers: { 'Content-Type': 'application/octet-stream' },
+        timeout: 30000
+      });
+    }
+
+    await api.post('/shards/register', {
+      shardId,
+      fileId,
+      order,
+      sizeBytes: encrypted.cipherText.length,
+      checksum,
+      nodeIds: plannedReplicas.map((replica) => replica.nodeId)
+    });
+
+    encryptionShardMeta.push({
+      shardId,
+      order,
+      iv: encrypted.iv,
+      authTag: encrypted.authTag
+    });
+  }
+
+  await api.post('/files/register', {
+    fileId,
+    originalName,
+    sizeBytes: plainBuffer.length,
+    shardCount: chunks.length,
+    cipher: 'aes-256-gcm',
+    metadata: {
+      encryption: {
+        algorithm: 'aes-256-gcm',
+        shardSizeBytes,
+        keyFormat: 'base64',
+        shards: encryptionShardMeta
+      }
+    }
+  });
+
+  console.log(`Upload complete`);
+  console.log(`fileId: ${fileId}`);
+  console.log(`originalName: ${originalName}`);
+  console.log(`sizeBytes: ${plainBuffer.length}`);
+  console.log(`shardCount: ${chunks.length}`);
+  console.log(`replicasPerShard: ${replicas}`);
+  console.log(`aes256KeyBase64: ${keyBase64}`);
+}
+
+async function downloadFile(options) {
+  const fileId = options.fileId;
+  const serverUrl = options.server;
+  const output = options.output ? path.resolve(options.output) : path.resolve(`./${fileId}.downloaded`);
+  const keyBuffer = parseAesKey(options.keyBase64);
+
+  const api = createApiClient(serverUrl);
+  const manifestResponse = await api.get(`/files/${fileId}/manifest`);
+  const manifest = manifestResponse.data;
+  const metadata = manifest.file?.metadata || {};
+  const encryption = metadata.encryption;
+
+  if (!encryption || encryption.algorithm !== 'aes-256-gcm') {
+    throw new Error('missing or unsupported encryption metadata');
+  }
+
+  const shardMetaMap = new Map((encryption.shards || []).map((entry) => [entry.shardId, entry]));
+
+  const orderedShards = [...(manifest.shards || [])].sort((a, b) => a.order - b.order);
+  const plainParts = [];
+
+  for (const shard of orderedShards) {
+    const shardMeta = shardMetaMap.get(shard.shardId);
+    if (!shardMeta) {
+      throw new Error(`missing encryption metadata for shard ${shard.shardId}`);
+    }
+
+    let encryptedBuffer = null;
+    for (const replica of shard.replicas || []) {
+      try {
+        const getUrl = `${normalizeUrl(replica.url)}/shards/${shard.shardId}`;
+        const response = await axios.get(getUrl, {
+          responseType: 'arraybuffer',
+          timeout: 30000
+        });
+        const candidate = Buffer.from(response.data);
+        if (sha256Hex(candidate) !== shard.checksum) {
+          continue;
+        }
+        encryptedBuffer = candidate;
+        break;
+      } catch {
+        continue;
+      }
+    }
+
+    if (!encryptedBuffer) {
+      throw new Error(`failed to fetch valid replica for shard ${shard.shardId}`);
+    }
+
+    const plain = decryptAes256Gcm(encryptedBuffer, keyBuffer, shardMeta.iv, shardMeta.authTag);
+    plainParts.push(plain);
+  }
+
+  const reconstructed = Buffer.concat(plainParts);
+  await fs.mkdir(path.dirname(output), { recursive: true });
+  await fs.writeFile(output, reconstructed);
+
+  console.log(`Download complete`);
+  console.log(`fileId: ${fileId}`);
+  console.log(`output: ${output}`);
+  console.log(`sizeBytes: ${reconstructed.length}`);
+}
+
+module.exports = {
+  uploadFile,
+  downloadFile
+};