nodio-cli 1.0.8 → 1.0.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nodio-cli",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Nodio distributed storage network",
   "main": "src/server/index.js",
   "type": "commonjs",

package/src/node/index.js

@@ -80,7 +80,8 @@ program
   .option('--capacity-gb <gb>', 'donated capacity in GB', '10')
   .option('--auto-port-start <port>', 'start of auto-port range', '5001')
   .option('--auto-port-end <port>', 'end of auto-port range', '5999')
-  .option('--heartbeat-ms <ms>', 'heartbeat interval in milliseconds', '30000');
+  .option('--heartbeat-ms <ms>', 'heartbeat interval in milliseconds', '30000')
+  .option('--relay-poll-ms <ms>', 'relay task poll interval in milliseconds', '1000');
 
 program.action(async (capacityArg, options) => {
   const autoPortStart = Number(options.autoPortStart);
@@ -92,6 +93,7 @@ program.action(async (capacityArg, options) => {
   const parsedCapacityArg = parseCapacityGb(capacityArg);
   const capacityGb = parsedCapacityArg ?? Number(options.capacityGb);
   const heartbeatMs = Number(options.heartbeatMs);
+  const relayPollMs = Number(options.relayPollMs);
   const advertisedHost = options.host === 'auto' ? detectAdvertisedHost() : options.host;
   const storageDir = options.storageDir
     ? path.resolve(options.storageDir)
@@ -109,6 +111,9 @@ program.action(async (capacityArg, options) => {
   if (!Number.isFinite(heartbeatMs) || heartbeatMs <= 0) {
     throw new Error('heartbeat-ms must be greater than 0');
   }
+  if (!Number.isFinite(relayPollMs) || relayPollMs <= 0) {
+    throw new Error('relay-poll-ms must be greater than 0');
+  }
 
   if (isLoopbackHost(advertisedHost)) {
     console.warn('[nodio-node] warning: loopback host is advertised; only this machine can reach this donor');
@@ -121,7 +126,8 @@ program.action(async (capacityArg, options) => {
     port,
     storageDir,
     capacityBytes: Math.floor(capacityGb * 1024 * 1024 * 1024),
-    heartbeatIntervalMs: heartbeatMs
+    heartbeatIntervalMs: heartbeatMs,
+    relayPollIntervalMs: relayPollMs
   });
 
   await runtime.start();

package/src/node/runtime.js

@@ -15,8 +15,11 @@ class NodioNodeRuntime {
     this.port = Number(options.port);
     this.capacityBytes = Number(options.capacityBytes);
     this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || 30000);
+    this.relayPollIntervalMs = Number(options.relayPollIntervalMs || 1000);
     this.shardStore = new LocalShardStore(options.storageDir);
     this.heartbeatTimer = null;
+    this.relayPollTimer = null;
+    this.relayPullInFlight = false;
   }
 
   async start() {
@@ -37,6 +40,14 @@ class NodioNodeRuntime {
         console.error('[heartbeat]', error.message);
       }
     }, this.heartbeatIntervalMs);
+
+    this.relayPollTimer = setInterval(async () => {
+      try {
+        await this.pollRelayTasks();
+      } catch (error) {
+        console.error('[relay-pull]', error.message);
+      }
+    }, this.relayPollIntervalMs);
   }
 
   async startShardServer() {
@@ -128,6 +139,11 @@ class NodioNodeRuntime {
       this.heartbeatIntervalMs = interval;
     }
 
+    const relayInterval = Number(response.data.relayPollIntervalMs);
+    if (Number.isFinite(relayInterval) && relayInterval > 0) {
+      this.relayPollIntervalMs = relayInterval;
+    }
+
     console.log(
       `Registered node ${this.nodeId} | min replicas: ${response.data.minReplicas} | emergency floor: ${response.data.emergencyReplicaFloor}`
     );
@@ -146,16 +162,36 @@ class NodioNodeRuntime {
       await this.executeReplicationTask(task);
     }
 
-    const relayTasks = response.data.relayTasks || [];
-    for (const task of relayTasks) {
-      await this.executeRelayTask(task);
-    }
-
     console.log(
-      `[heartbeat] ${new Date().toISOString()} | shards=${shardIds.length} | tasks=${tasks.length} | relayTasks=${relayTasks.length}`
+      `[heartbeat] ${new Date().toISOString()} | shards=${shardIds.length} | tasks=${tasks.length}`
     );
   }
 
+  async pollRelayTasks() {
+    if (this.relayPullInFlight) {
+      return;
+    }
+
+    this.relayPullInFlight = true;
+    try {
+      const response = await axios.post(`${this.serverUrl}/api/nodes/relay-pull`, {
+        nodeId: this.nodeId
+      });
+
+      const relayTasks = response.data.relayTasks || [];
+      for (const task of relayTasks) {
+        // eslint-disable-next-line no-await-in-loop
+        await this.executeRelayTask(task);
+      }
+
+      if (relayTasks.length > 0) {
+        console.log(`[relay-pull] ${new Date().toISOString()} | relayTasks=${relayTasks.length}`);
+      }
+    } finally {
+      this.relayPullInFlight = false;
+    }
+  }
+
   async executeReplicationTask(task) {
     const sourceShardUrl = `${normalizeUrl(task.sourceUrl)}/shards/${task.shardId}`;
     try {

package/src/server/config.js

@@ -6,6 +6,7 @@ function getServerConfig() {
     port: Number(process.env.PORT || process.env.NODIO_SERVER_PORT || 4000),
     mongoUri: process.env.NODIO_MONGO_URI || 'mongodb://127.0.0.1:27017/nodio',
     heartbeatIntervalMs: Number(process.env.NODIO_HEARTBEAT_INTERVAL_MS || 10000),
+    relayPollIntervalMs: Number(process.env.NODIO_RELAY_POLL_INTERVAL_MS || 1000),
     offlineAfterMisses: Number(process.env.NODIO_OFFLINE_AFTER_MISSES || 3),
     minReplicas: Number(process.env.NODIO_MIN_REPLICAS || 5),
     emergencyReplicaFloor: Number(process.env.NODIO_EMERGENCY_REPLICA_FLOOR || 2)

package/src/server/routes.js

@@ -23,6 +23,31 @@ function normalizeUrl(url) {
   return String(url || '').replace(/\/+$/, '');
 }
 
+async function claimPendingRelayTasks(nodeId, limit = 10) {
+  const pendingRelayTasks = await RelayTaskModel.find({
+    nodeId,
+    status: 'pending'
+  })
+    .sort({ createdAt: 1 })
+    .limit(limit)
+    .lean();
+
+  if (pendingRelayTasks.length > 0) {
+    await RelayTaskModel.updateMany(
+      { _id: { $in: pendingRelayTasks.map((task) => task._id) } },
+      { $set: { status: 'in_progress' }, $inc: { attempts: 1 } }
+    );
+  }
+
+  return pendingRelayTasks.map((task) => ({
+    taskId: task._id.toString(),
+    taskType: task.taskType,
+    shardId: task.shardId,
+    fileId: task.fileId,
+    dataBase64: task.dataBase64
+  }));
+}
+
 function buildRoutes(config) {
   const router = express.Router();
 
@@ -135,6 +160,7 @@ function buildRoutes(config) {
         nodeId: node.nodeId,
         status: node.status,
         heartbeatIntervalMs: config.heartbeatIntervalMs,
+        relayPollIntervalMs: config.relayPollIntervalMs,
         minReplicas: config.minReplicas,
         emergencyReplicaFloor: config.emergencyReplicaFloor
       });
@@ -237,33 +263,40 @@ function buildRoutes(config) {
         });
       }
 
-      const pendingRelayTasks = await RelayTaskModel.find({
-        nodeId,
-        status: 'pending'
-      })
-        .sort({ createdAt: 1 })
-        .limit(10)
-        .lean();
+      res.json({
+        ok: true,
+        now: new Date().toISOString(),
+        replicationTasks: tasksWithSourceUrl,
+        relayTasks: []
+      });
+    } catch (error) {
+      next(error);
+    }
+  });
 
-      if (pendingRelayTasks.length > 0) {
-        await RelayTaskModel.updateMany(
-          { _id: { $in: pendingRelayTasks.map((task) => task._id) } },
-          { $set: { status: 'in_progress' }, $inc: { attempts: 1 } }
-        );
+  router.post('/nodes/relay-pull', async (req, res, next) => {
+    try {
+      const { nodeId } = req.body;
+      if (!nodeId) {
+        return res.status(400).json({ error: 'nodeId is required' });
       }
 
-      const relayTasks = pendingRelayTasks.map((task) => ({
-        taskId: task._id.toString(),
-        taskType: task.taskType,
-        shardId: task.shardId,
-        fileId: task.fileId,
-        dataBase64: task.dataBase64
-      }));
+      const node = await NodeModel.findOne({ nodeId });
+      if (!node) {
+        return res.status(404).json({ error: 'node not found' });
+      }
+
+      const relayTasks = await claimPendingRelayTasks(nodeId, 10);
+
+      if (relayTasks.length > 0 && node.pendingRelayAlert) {
+        node.pendingRelayAlert = false;
+        node.pendingRelayAlertAt = null;
+        await node.save();
+      }
 
       res.json({
         ok: true,
         now: new Date().toISOString(),
-        replicationTasks: tasksWithSourceUrl,
         relayTasks
       });
     } catch (error) {

package/src/user/commands.js

@@ -31,7 +31,7 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-async function relayStoreShard(api, { shardId, fileId, nodeIds, dataBuffer, timeoutMs = 90000, pollMs = 500 }) {
+async function relayStoreShard(api, { shardId, fileId, nodeIds, dataBuffer, timeoutMs = 45000, pollMs = 200 }) {
   const opId = uuidv4();
   await api.post('/relay/shards/store', {
     opId,
@@ -42,14 +42,7 @@ async function relayStoreShard(api, { shardId, fileId, nodeIds, dataBuffer, time
   });
 
   // Alert server that relay tasks are pending so donors check urgently
-  try {
-    for (const nodeId of nodeIds) {
-      // eslint-disable-next-line no-await-in-loop
-      await api.post(`/nodes/${nodeId}/alert-relay-pending`);
-    }
-  } catch (alertError) {
-    console.warn('relay alert failed:', alertError.message);
-  }
+  await Promise.allSettled(nodeIds.map((nodeId) => api.post(`/nodes/${nodeId}/alert-relay-pending`)));
 
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
@@ -69,7 +62,7 @@ async function relayStoreShard(api, { shardId, fileId, nodeIds, dataBuffer, time
   throw new Error(`relay store timed out for shard ${shardId}`);
 }
 
-async function relayFetchShard(api, { shardId, nodeIds, timeoutMs = 90000, pollMs = 500 }) {
+async function relayFetchShard(api, { shardId, nodeIds, timeoutMs = 45000, pollMs = 200 }) {
   const opId = uuidv4();
   await api.post('/relay/shards/fetch', {
     opId,
@@ -78,14 +71,7 @@ async function relayFetchShard(api, { shardId, nodeIds, timeoutMs = 90000, pollM
   });
 
   // Alert server that relay tasks are pending so donors check urgently
-  try {
-    for (const nodeId of nodeIds) {
-      // eslint-disable-next-line no-await-in-loop
-      await api.post(`/nodes/${nodeId}/alert-relay-pending`);
-    }
-  } catch (alertError) {
-    console.warn('relay alert failed:', alertError.message);
-  }
+  await Promise.allSettled(nodeIds.map((nodeId) => api.post(`/nodes/${nodeId}/alert-relay-pending`)));
 
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
@@ -120,6 +106,8 @@ async function uploadFile(options) {
   const serverUrl = options.server;
   const shardSizeMb = Number(options.shardSizeMb || 1);
   const replicas = Number(options.replicas || 5);
+  const directTimeoutMs = Number(options.directTimeoutMs || 1200);
+  const relayFirst = Boolean(options.relayFirst);
 
   if (!Number.isFinite(shardSizeMb) || shardSizeMb <= 0) {
     throw new Error('shard-size-mb must be greater than 0');
@@ -127,6 +115,9 @@ async function uploadFile(options) {
   if (!Number.isInteger(replicas) || replicas < 5) {
     throw new Error('replicas must be an integer >= 5');
   }
+  if (!Number.isFinite(directTimeoutMs) || directTimeoutMs <= 0) {
+    throw new Error('direct-timeout-ms must be greater than 0');
+  }
 
   const shardSizeBytes = Math.floor(shardSizeMb * 1024 * 1024);
   const plainBuffer = await fs.readFile(filePath);
@@ -173,20 +164,39 @@ async function uploadFile(options) {
       throw new Error(`placement failed for ${shardId}: expected ${replicas} replicas`);
     }
 
-    const successfulReplicas = [];
-    const failedReplicas = [];
-
-    for (const replica of plannedReplicas) {
-      try {
-        const putUrl = `${normalizeUrl(replica.url)}/shards/${shardId}`;
-        await axios.put(putUrl, encrypted.cipherText, {
-          headers: { 'Content-Type': 'application/octet-stream' },
-          timeout: 30000
-        });
-        successfulReplicas.push(replica);
-      } catch (error) {
-        failedReplicas.push({ nodeId: replica.nodeId, url: replica.url, error: error.message });
-      }
+    let successfulReplicas = [];
+    let failedReplicas = [];
+
+    if (relayFirst) {
+      failedReplicas = plannedReplicas.map((replica) => ({
+        nodeId: replica.nodeId,
+        url: replica.url,
+        error: 'direct write skipped (relay-first)'
+      }));
+    } else {
+      const directWriteResults = await Promise.allSettled(
+        plannedReplicas.map(async (replica) => {
+          const putUrl = `${normalizeUrl(replica.url)}/shards/${shardId}`;
+          await axios.put(putUrl, encrypted.cipherText, {
+            headers: { 'Content-Type': 'application/octet-stream' },
+            timeout: directTimeoutMs
+          });
+          return replica;
+        })
+      );
+
+      successfulReplicas = directWriteResults
+        .filter((result) => result.status === 'fulfilled')
+        .map((result) => result.value);
+
+      failedReplicas = directWriteResults
+        .map((result, index) => ({ result, replica: plannedReplicas[index] }))
+        .filter((entry) => entry.result.status === 'rejected')
+        .map((entry) => ({
+          nodeId: entry.replica.nodeId,
+          url: entry.replica.url,
+          error: entry.result.reason?.message || 'direct write failed'
+        }));
     }
 
     if (failedReplicas.length > 0) {
@@ -260,6 +270,12 @@ async function downloadFile(options) {
   const serverUrl = options.server;
   const output = options.output ? path.resolve(options.output) : path.resolve(`./${fileId}.downloaded`);
   const keyBuffer = parseAesKey(options.keyBase64);
+  const directTimeoutMs = Number(options.directTimeoutMs || 1200);
+  const relayFirst = Boolean(options.relayFirst);
+
+  if (!Number.isFinite(directTimeoutMs) || directTimeoutMs <= 0) {
+    throw new Error('direct-timeout-ms must be greater than 0');
+  }
 
   const api = createApiClient(serverUrl);
   const manifestResponse = await api.get(`/files/${fileId}/manifest`);
@@ -283,21 +299,25 @@ async function downloadFile(options) {
     }
 
     let encryptedBuffer = null;
-    for (const replica of shard.replicas || []) {
-      try {
-        const getUrl = `${normalizeUrl(replica.url)}/shards/${shard.shardId}`;
-        const response = await axios.get(getUrl, {
-          responseType: 'arraybuffer',
-          timeout: 30000
-        });
-        const candidate = Buffer.from(response.data);
-        if (sha256Hex(candidate) !== shard.checksum) {
-          continue;
-        }
-        encryptedBuffer = candidate;
-        break;
-      } catch {
-        continue;
+    if (!relayFirst) {
+      const directFetchResults = await Promise.allSettled(
+        (shard.replicas || []).map(async (replica) => {
+          const getUrl = `${normalizeUrl(replica.url)}/shards/${shard.shardId}`;
+          const response = await axios.get(getUrl, {
+            responseType: 'arraybuffer',
+            timeout: directTimeoutMs
+          });
+          const candidate = Buffer.from(response.data);
+          if (sha256Hex(candidate) !== shard.checksum) {
+            throw new Error('checksum mismatch');
+          }
+          return candidate;
+        })
+      );
+
+      const successfulDirectFetch = directFetchResults.find((result) => result.status === 'fulfilled');
+      if (successfulDirectFetch) {
+        encryptedBuffer = successfulDirectFetch.value;
       }
     }
 

package/src/user/index.js

@@ -14,6 +14,8 @@ program
   .option('--file-id <id>', 'custom file ID (optional)')
   .option('--shard-size-mb <mb>', 'plaintext shard size in MB', '1')
   .option('--replicas <count>', 'replicas per shard (minimum 5)', '5')
+  .option('--direct-timeout-ms <ms>', 'timeout for each direct donor attempt before relay fallback', '1200')
+  .option('--relay-first', 'skip direct donor attempts and use relay path immediately')
   .option('--key-base64 <key>', '32-byte AES key in base64 (optional)')
   .action(async (options) => {
     await uploadFile(options);
@@ -26,6 +28,8 @@ program
   .requiredOption('--key-base64 <key>', '32-byte AES key in base64 from upload output')
   .option('--server <url>', 'central server URL', 'https://api.nodio.me')
   .option('--output <path>', 'output file path')
+  .option('--direct-timeout-ms <ms>', 'timeout for each direct donor attempt before relay fallback', '1200')
+  .option('--relay-first', 'skip direct donor attempts and use relay path immediately')
   .action(async (options) => {
     await downloadFile(options);
   });