nodio-cli 1.0.0

package/.env.example

@@ -0,0 +1,6 @@
+NODIO_SERVER_PORT=4000
+NODIO_MONGO_URI=mongodb://127.0.0.1:27017/nodio
+NODIO_HEARTBEAT_INTERVAL_MS=30000
+NODIO_OFFLINE_AFTER_MISSES=3
+NODIO_MIN_REPLICAS=5
+NODIO_EMERGENCY_REPLICA_FLOOR=2

package/README.md

@@ -0,0 +1,127 @@
+# Nodio-CLI
+
+Nodio is a CLI-based distributed storage network.
+
+This implementation includes:
+- Central server (Express + MongoDB) for node registry, shard metadata, placement planning, heartbeat tracking, and replication task orchestration
+- `nodio-node` donor CLI for running storage nodes, sending heartbeats every 30s, storing shard blobs, and executing replication tasks
+- `nodio` user CLI for encrypted upload/download with shard distribution and reconstruction
+
+## Requirements
+
+- Node.js 20+
+- MongoDB running locally or reachable by URI
+
+## Install
+
+```bash
+npm install
+```
+
+## Configure
+
+```bash
+cp .env.example .env
+```
+
+Environment values:
+- `NODIO_SERVER_PORT`: central API port (default `4000`)
+- `NODIO_MONGO_URI`: MongoDB connection string
+- `NODIO_HEARTBEAT_INTERVAL_MS`: expected heartbeat interval (`30000`)
+- `NODIO_OFFLINE_AFTER_MISSES`: offline threshold in missed heartbeats (`3`)
+- `NODIO_MIN_REPLICAS`: target replicas for placement planning (`5`)
+- `NODIO_EMERGENCY_REPLICA_FLOOR`: immediate repair threshold (`2`)
+
+## Start Central Server
+
+```bash
+npm run start:server
+```
+
+Server API base: `http://127.0.0.1:4000/api`
+
+## Deploy On Render (Central Server)
+
+Use a **Web Service** for the central server.
+
+- Build Command: `npm install`
+- Start Command: `npm run start:server`
+- Health Check Path: `/api/health`
+
+Required environment variables on Render:
+- `NODIO_MONGO_URI` (MongoDB Atlas URI)
+
+Optional environment variables:
+- `NODIO_HEARTBEAT_INTERVAL_MS`
+- `NODIO_OFFLINE_AFTER_MISSES`
+- `NODIO_MIN_REPLICAS`
+- `NODIO_EMERGENCY_REPLICA_FLOOR`
+
+Notes:
+- Render sets `PORT` automatically, and the server now uses it.
+- Donor nodes (`nodio-node`) should run on persistent machines or VMs, not on ephemeral web-service instances.
+
+## Start Donor Node CLI
+
+```bash
+npm run start:node -- \
+	--node-id node-a \
+	--server http://127.0.0.1:4000 \
+	--host 127.0.0.1 \
+	--port 5001 \
+	--storage-dir ./.nodio-node-a \
+	--capacity-gb 10
+```
+
+Run more nodes by changing `--node-id`, `--port`, and `--storage-dir`.
+
+## User CLI (Phase 2)
+
+Upload (encrypt + shard + distribute):
+
+```bash
+npm run start:cli -- upload \
+	--file ./example.txt \
+	--server http://127.0.0.1:4000 \
+	--shard-size-mb 1 \
+	--replicas 5
+```
+
+The upload command prints:
+- `fileId`
+- `aes256KeyBase64` (required for download)
+
+Download (fetch + verify + decrypt + reconstruct):
+
+```bash
+npm run start:cli -- download \
+	--file-id <FILE_ID> \
+	--key-base64 <AES_256_KEY_BASE64> \
+	--server http://127.0.0.1:4000 \
+	--output ./restored-example.txt
+```
+
+## Core Behavior Implemented
+
+- Files can be represented as shards with metadata in MongoDB (`files`, `shards`, and `shard placements`)
+- Placement planning enforces distinct nodes per shard and defaults to 5 replicas
+- User uploads enforce at least 5 replicas per shard
+- Node heartbeats every 30 seconds update status and available storage
+- If a node misses 3 heartbeat intervals, it is marked offline
+- When live replicas of a shard drop below 2, the server immediately creates replication tasks to healthy nodes
+- Donor nodes fetch pending replication tasks through heartbeats and self-heal by copying shard data from source nodes
+- User downloads verify shard checksums, decrypt with AES-256-GCM metadata, and reconstruct files in shard order
+
+## Implemented API Surface
+
+- `POST /api/nodes/register`
+- `POST /api/nodes/heartbeat`
+- `POST /api/replication-tasks/:taskId/complete`
+- `POST /api/files/register`
+- `POST /api/shards/register`
+- `POST /api/shards/placement-plan`
+- `GET /api/files/:fileId/manifest`
+
+## Next Step
+
+Add authentication between user CLI, central server, and donor nodes for production security.

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "nodio-cli",
+  "version": "1.0.0",
+  "description": "Nodio distributed storage network",
+  "main": "src/server/index.js",
+  "type": "commonjs",
+  "bin": {
+    "nodio-server": "src/server/index.js",
+    "nodio-node": "src/node/index.js",
+    "nodio": "src/user/index.js"
+  },
+  "scripts": {
+    "start:server": "node ./src/server/index.js",
+    "start:node": "node ./src/node/index.js",
+    "start:cli": "node ./src/user/index.js",
+    "lint": "node -e \"console.log('No linter configured')\""
+  },
+  "keywords": [
+    "distributed-storage",
+    "cli",
+    "nodio"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.11.0",
+    "commander": "^14.0.1",
+    "dotenv": "^17.2.2",
+    "express": "^5.1.0",
+    "mongoose": "^8.18.0",
+    "uuid": "^13.0.0"
+  }
+}

package/render.yaml

@@ -0,0 +1,22 @@
+services:
+  - type: web
+    name: nodio-server
+    env: node
+    rootDir: .
+    buildCommand: npm install
+    startCommand: npm run start:server
+    healthCheckPath: /api/health
+    autoDeploy: true
+    envVars:
+      - key: NODE_VERSION
+        value: 20
+      - key: NODIO_MONGO_URI
+        sync: false
+      - key: NODIO_HEARTBEAT_INTERVAL_MS
+        value: "30000"
+      - key: NODIO_OFFLINE_AFTER_MISSES
+        value: "3"
+      - key: NODIO_MIN_REPLICAS
+        value: "5"
+      - key: NODIO_EMERGENCY_REPLICA_FLOOR
+        value: "2"

package/src/common/crypto.js

@@ -0,0 +1,41 @@
+const crypto = require('crypto');
+
+function sha256Hex(buffer) {
+  return crypto.createHash('sha256').update(buffer).digest('hex');
+}
+
+function encryptAes256Gcm(plainBuffer, keyBuffer) {
+  if (!Buffer.isBuffer(keyBuffer) || keyBuffer.length !== 32) {
+    throw new Error('keyBuffer must be a 32-byte buffer for AES-256');
+  }
+
+  const iv = crypto.randomBytes(12);
+  const cipher = crypto.createCipheriv('aes-256-gcm', keyBuffer, iv);
+  const encrypted = Buffer.concat([cipher.update(plainBuffer), cipher.final()]);
+  const authTag = cipher.getAuthTag();
+  return {
+    cipherText: encrypted,
+    iv: iv.toString('base64'),
+    authTag: authTag.toString('base64')
+  };
+}
+
+function decryptAes256Gcm(cipherBuffer, keyBuffer, ivBase64, authTagBase64) {
+  if (!Buffer.isBuffer(keyBuffer) || keyBuffer.length !== 32) {
+    throw new Error('keyBuffer must be a 32-byte buffer for AES-256');
+  }
+
+  const decipher = crypto.createDecipheriv(
+    'aes-256-gcm',
+    keyBuffer,
+    Buffer.from(ivBase64, 'base64')
+  );
+  decipher.setAuthTag(Buffer.from(authTagBase64, 'base64'));
+  return Buffer.concat([decipher.update(cipherBuffer), decipher.final()]);
+}
+
+module.exports = {
+  sha256Hex,
+  encryptAes256Gcm,
+  decryptAes256Gcm
+};

package/src/node/index.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+const path = require('path');
+const os = require('os');
+const { Command } = require('commander');
+const { v4: uuidv4 } = require('uuid');
+const { NodioNodeRuntime } = require('./runtime');
+
+const program = new Command();
+
+program
+  .name('nodio-node')
+  .description('Nodio donor node CLI')
+  .option('--node-id <id>', 'unique node ID', uuidv4())
+  .option('--server <url>', 'central server URL', 'http://127.0.0.1:4000')
+  .option('--host <host>', 'host/IP exposed to network', '127.0.0.1')
+  .option('--port <port>', 'port to expose shard API', '5001')
+  .option('--storage-dir <path>', 'local shard storage directory', path.join(os.homedir(), '.nodio-node'))
+  .option('--capacity-gb <gb>', 'donated capacity in GB', '10')
+  .option('--heartbeat-ms <ms>', 'heartbeat interval in milliseconds', '30000');
+
+program.action(async (options) => {
+  const port = Number(options.port);
+  const capacityGb = Number(options.capacityGb);
+  const heartbeatMs = Number(options.heartbeatMs);
+
+  if (!Number.isInteger(port) || port <= 0) {
+    throw new Error('port must be a positive integer');
+  }
+  if (!Number.isFinite(capacityGb) || capacityGb <= 0) {
+    throw new Error('capacity-gb must be greater than 0');
+  }
+  if (!Number.isFinite(heartbeatMs) || heartbeatMs <= 0) {
+    throw new Error('heartbeat-ms must be greater than 0');
+  }
+
+  const runtime = new NodioNodeRuntime({
+    nodeId: options.nodeId,
+    serverUrl: options.server,
+    publicUrl: `http://${options.host}:${port}`,
+    port,
+    storageDir: path.resolve(options.storageDir),
+    capacityBytes: Math.floor(capacityGb * 1024 * 1024 * 1024),
+    heartbeatIntervalMs: heartbeatMs
+  });
+
+  await runtime.start();
+});
+
+program.parseAsync(process.argv).catch((error) => {
+  console.error('[nodio-node]', error.message);
+  process.exit(1);
+});

package/src/node/runtime.js

@@ -0,0 +1,154 @@
+const express = require('express');
+const axios = require('axios');
+const { LocalShardStore } = require('./storage');
+
+function normalizeUrl(url) {
+  return String(url || '').replace(/\/+$/, '');
+}
+
+class NodioNodeRuntime {
+  constructor(options) {
+    this.nodeId = options.nodeId;
+    this.serverUrl = normalizeUrl(options.serverUrl);
+    this.publicUrl = normalizeUrl(options.publicUrl);
+    this.port = Number(options.port);
+    this.capacityBytes = Number(options.capacityBytes);
+    this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || 30000);
+    this.shardStore = new LocalShardStore(options.storageDir);
+    this.heartbeatTimer = null;
+  }
+
+  async start() {
+    await this.shardStore.init();
+    await this.startShardServer();
+    await this.registerNode();
+    await this.sendHeartbeat();
+
+    this.heartbeatTimer = setInterval(async () => {
+      try {
+        await this.sendHeartbeat();
+      } catch (error) {
+        console.error('[heartbeat]', error.message);
+      }
+    }, this.heartbeatIntervalMs);
+  }
+
+  async startShardServer() {
+    const app = express();
+    app.use('/shards/:shardId', express.raw({ type: '*/*', limit: '200mb' }));
+
+    app.put('/shards/:shardId', async (req, res) => {
+      try {
+        const { shardId } = req.params;
+        if (!Buffer.isBuffer(req.body)) {
+          return res.status(400).json({ error: 'binary payload is required' });
+        }
+        const record = await this.shardStore.saveShard(shardId, req.body);
+        res.json({ ok: true, shard: record });
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    app.get('/shards/:shardId', async (req, res) => {
+      try {
+        const { shardId } = req.params;
+        if (!(await this.shardStore.hasShard(shardId))) {
+          return res.status(404).json({ error: 'shard not found' });
+        }
+
+        const data = await this.shardStore.readShard(shardId);
+        res.setHeader('Content-Type', 'application/octet-stream');
+        res.send(data);
+      } catch (error) {
+        res.status(500).json({ error: error.message });
+      }
+    });
+
+    app.get('/health', (_req, res) => {
+      res.json({ ok: true, nodeId: this.nodeId });
+    });
+
+    await new Promise((resolve) => {
+      app.listen(this.port, () => {
+        console.log(`Nodio node ${this.nodeId} listening on port ${this.port}`);
+        resolve();
+      });
+    });
+  }
+
+  async freeBytes() {
+    const used = await this.shardStore.usedBytes();
+    return Math.max(this.capacityBytes - used, 0);
+  }
+
+  async registerNode() {
+    const response = await axios.post(`${this.serverUrl}/api/nodes/register`, {
+      nodeId: this.nodeId,
+      url: this.publicUrl,
+      capacityBytes: this.capacityBytes,
+      freeBytes: await this.freeBytes()
+    });
+
+    const interval = Number(response.data.heartbeatIntervalMs);
+    if (Number.isFinite(interval) && interval > 0) {
+      this.heartbeatIntervalMs = interval;
+    }
+
+    console.log(
+      `Registered node ${this.nodeId} | min replicas: ${response.data.minReplicas} | emergency floor: ${response.data.emergencyReplicaFloor}`
+    );
+  }
+
+  async sendHeartbeat() {
+    const shardIds = await this.shardStore.listShardIds();
+    const response = await axios.post(`${this.serverUrl}/api/nodes/heartbeat`, {
+      nodeId: this.nodeId,
+      freeBytes: await this.freeBytes(),
+      shardIds
+    });
+
+    const tasks = response.data.replicationTasks || [];
+    for (const task of tasks) {
+      await this.executeReplicationTask(task);
+    }
+
+    console.log(
+      `[heartbeat] ${new Date().toISOString()} | shards=${shardIds.length} | tasks=${tasks.length}`
+    );
+  }
+
+  async executeReplicationTask(task) {
+    const sourceShardUrl = `${normalizeUrl(task.sourceUrl)}/shards/${task.shardId}`;
+    try {
+      const sourceResponse = await axios.get(sourceShardUrl, {
+        responseType: 'arraybuffer',
+        timeout: 30000
+      });
+      const data = Buffer.from(sourceResponse.data);
+      await this.shardStore.saveShard(task.shardId, data);
+
+      await axios.post(`${this.serverUrl}/api/replication-tasks/${task.taskId}/complete`, {
+        nodeId: this.nodeId,
+        success: true
+      });
+
+      console.log(`[replication] completed task=${task.taskId} shard=${task.shardId}`);
+    } catch (error) {
+      const message = error.response?.data?.error || error.message;
+      await axios
+        .post(`${this.serverUrl}/api/replication-tasks/${task.taskId}/complete`, {
+          nodeId: this.nodeId,
+          success: false,
+          errorMessage: message
+        })
+        .catch(() => null);
+
+      console.error(`[replication] failed task=${task.taskId} shard=${task.shardId}: ${message}`);
+    }
+  }
+}
+
+module.exports = {
+  NodioNodeRuntime
+};

package/src/node/storage.js

@@ -0,0 +1,80 @@
+const fs = require('fs/promises');
+const path = require('path');
+const { sha256Hex } = require('../common/crypto');
+
+class LocalShardStore {
+  constructor(baseDir) {
+    this.baseDir = baseDir;
+    this.indexFile = path.join(baseDir, 'index.json');
+    this.shardsDir = path.join(baseDir, 'shards');
+  }
+
+  async init() {
+    await fs.mkdir(this.shardsDir, { recursive: true });
+    try {
+      await fs.access(this.indexFile);
+    } catch {
+      await fs.writeFile(this.indexFile, JSON.stringify({ shards: {} }, null, 2), 'utf-8');
+    }
+  }
+
+  async readIndex() {
+    const raw = await fs.readFile(this.indexFile, 'utf-8');
+    const parsed = JSON.parse(raw);
+    if (!parsed.shards || typeof parsed.shards !== 'object') {
+      return { shards: {} };
+    }
+    return parsed;
+  }
+
+  async writeIndex(index) {
+    await fs.writeFile(this.indexFile, JSON.stringify(index, null, 2), 'utf-8');
+  }
+
+  shardPath(shardId) {
+    return path.join(this.shardsDir, `${shardId}.bin`);
+  }
+
+  async saveShard(shardId, dataBuffer) {
+    const filePath = this.shardPath(shardId);
+    await fs.writeFile(filePath, dataBuffer);
+
+    const index = await this.readIndex();
+    index.shards[shardId] = {
+      shardId,
+      sizeBytes: dataBuffer.byteLength,
+      checksum: sha256Hex(dataBuffer),
+      updatedAt: new Date().toISOString()
+    };
+    await this.writeIndex(index);
+    return index.shards[shardId];
+  }
+
+  async readShard(shardId) {
+    const filePath = this.shardPath(shardId);
+    return fs.readFile(filePath);
+  }
+
+  async hasShard(shardId) {
+    try {
+      await fs.access(this.shardPath(shardId));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async listShardIds() {
+    const index = await this.readIndex();
+    return Object.keys(index.shards);
+  }
+
+  async usedBytes() {
+    const index = await this.readIndex();
+    return Object.values(index.shards).reduce((sum, shard) => sum + Number(shard.sizeBytes || 0), 0);
+  }
+}
+
+module.exports = {
+  LocalShardStore
+};

package/src/server/config.js

@@ -0,0 +1,17 @@
+const path = require('path');
+require('dotenv').config({ path: path.resolve(process.cwd(), '.env') });
+
+function getServerConfig() {
+  return {
+    port: Number(process.env.PORT || process.env.NODIO_SERVER_PORT || 4000),
+    mongoUri: process.env.NODIO_MONGO_URI || 'mongodb://127.0.0.1:27017/nodio',
+    heartbeatIntervalMs: Number(process.env.NODIO_HEARTBEAT_INTERVAL_MS || 30000),
+    offlineAfterMisses: Number(process.env.NODIO_OFFLINE_AFTER_MISSES || 3),
+    minReplicas: Number(process.env.NODIO_MIN_REPLICAS || 5),
+    emergencyReplicaFloor: Number(process.env.NODIO_EMERGENCY_REPLICA_FLOOR || 2)
+  };
+}
+
+module.exports = {
+  getServerConfig
+};

package/src/server/index.js

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+const express = require('express');
+const mongoose = require('mongoose');
+const { getServerConfig } = require('./config');
+const { buildRoutes } = require('./routes');
+const { NodeModel } = require('./models');
+const { markNodeOfflineAndRecover } = require('./services');
+
+async function startServer() {
+  const config = getServerConfig();
+
+  await mongoose.connect(config.mongoUri);
+
+  const app = express();
+  app.use(express.json({ limit: '10mb' }));
+  app.use('/api', buildRoutes(config));
+
+  app.use((error, _req, res, _next) => {
+    const status = error.statusCode || 500;
+    const message = error.message || 'internal server error';
+    if (status >= 500) {
+      console.error('[server-error]', error);
+    }
+    res.status(status).json({ error: message });
+  });
+
+  const offlineThresholdMs = config.heartbeatIntervalMs * config.offlineAfterMisses;
+
+  setInterval(async () => {
+    try {
+      const staleBefore = new Date(Date.now() - offlineThresholdMs);
+      const staleNodes = await NodeModel.find({
+        status: 'online',
+        lastHeartbeatAt: { $lt: staleBefore }
+      })
+        .select('nodeId')
+        .lean();
+
+      for (const node of staleNodes) {
+        const changed = await markNodeOfflineAndRecover(node.nodeId, config.emergencyReplicaFloor);
+        if (changed) {
+          console.warn(`[heartbeat] node ${node.nodeId} marked offline`);
+        }
+      }
+    } catch (error) {
+      console.error('[offline-monitor]', error);
+    }
+  }, config.heartbeatIntervalMs);
+
+  app.listen(config.port, () => {
+    console.log(`Nodio central server listening on port ${config.port}`);
+  });
+}
+
+startServer().catch((error) => {
+  console.error('[startup]', error);
+  process.exit(1);
+});

package/src/server/models.js

@@ -0,0 +1,84 @@
+const mongoose = require('mongoose');
+
+const nodeSchema = new mongoose.Schema(
+  {
+    nodeId: { type: String, required: true, unique: true, index: true },
+    url: { type: String, required: true },
+    capacityBytes: { type: Number, required: true, min: 1 },
+    freeBytes: { type: Number, required: true, min: 0 },
+    status: { type: String, enum: ['online', 'offline'], default: 'online', index: true },
+    lastHeartbeatAt: { type: Date, default: Date.now, index: true }
+  },
+  { timestamps: true }
+);
+
+const fileSchema = new mongoose.Schema(
+  {
+    fileId: { type: String, required: true, unique: true, index: true },
+    originalName: { type: String, required: true },
+    sizeBytes: { type: Number, required: true, min: 0 },
+    shardCount: { type: Number, required: true, min: 1 },
+    cipher: { type: String, required: true, default: 'aes-256-gcm' },
+    metadata: { type: mongoose.Schema.Types.Mixed, default: {} }
+  },
+  { timestamps: true }
+);
+
+const shardSchema = new mongoose.Schema(
+  {
+    shardId: { type: String, required: true, unique: true, index: true },
+    fileId: { type: String, required: true, index: true },
+    order: { type: Number, required: true, min: 0 },
+    sizeBytes: { type: Number, required: true, min: 0 },
+    checksum: { type: String, required: true }
+  },
+  { timestamps: true }
+);
+
+const shardPlacementSchema = new mongoose.Schema(
+  {
+    shardId: { type: String, required: true, index: true },
+    fileId: { type: String, required: true, index: true },
+    nodeId: { type: String, required: true, index: true },
+    status: { type: String, enum: ['available', 'lost'], default: 'available', index: true }
+  },
+  { timestamps: true }
+);
+
+shardPlacementSchema.index({ shardId: 1, nodeId: 1 }, { unique: true });
+
+const replicationTaskSchema = new mongoose.Schema(
+  {
+    shardId: { type: String, required: true, index: true },
+    fileId: { type: String, required: true, index: true },
+    sourceNodeId: { type: String, required: true, index: true },
+    targetNodeId: { type: String, required: true, index: true },
+    status: {
+      type: String,
+      enum: ['pending', 'in_progress', 'completed', 'failed'],
+      default: 'pending',
+      index: true
+    },
+    errorMessage: { type: String, default: null },
+    attempts: { type: Number, default: 0 }
+  },
+  { timestamps: true }
+);
+
+replicationTaskSchema.index(
+  { shardId: 1, targetNodeId: 1, status: 1 },
+  {
+    unique: true,
+    partialFilterExpression: {
+      status: { $in: ['pending', 'in_progress'] }
+    }
+  }
+);
+
+module.exports = {
+  NodeModel: mongoose.model('Node', nodeSchema),
+  FileModel: mongoose.model('File', fileSchema),
+  ShardModel: mongoose.model('Shard', shardSchema),
+  ShardPlacementModel: mongoose.model('ShardPlacement', shardPlacementSchema),
+  ReplicationTaskModel: mongoose.model('ReplicationTask', replicationTaskSchema)
+};