cbyte 1.4.0 → 1.5.0

package/cbyte.cjs

@@ -0,0 +1,189 @@
+#!/usr/bin/env node
+
+// Increase listener limit to avoid warnings
+require('events').defaultMaxListeners = 6000;
+
+import fs from 'fs';
+import crypto from 'crypto';
+import { Command } from 'commander';
+import { createHelia } from 'helia';
+import * as Block from 'multiformats/block';
+import * as rawCodec from 'multiformats/codecs/raw';
+import * as sha256 from 'multiformats/hashes/sha2';
+import os from 'os';
+import path from 'path';
+
+const program = new Command();
+let helia = null;
+
+// Initialize Helia node once
+async function ensureHelia() {
+  if (!helia) {
+    helia = await createHelia();
+  }
+}
+
+// Simple SHA‑256 progress bar
+function renderProgress(completed, total, label = '') {
+  const width = 40;
+  const ratio = completed / total;
+  const filled = Math.floor(ratio * width);
+  const bar = '='.repeat(filled) + ' '.repeat(width - filled);
+  process.stderr.write(`\r${label} [${bar}] ${(ratio * 100).toFixed(1)}% (${completed}/${total})`);
+}
+
+// Pack operation with Helia
+async function pack(filePath, outFile, chunkSize = 4096, pin = false) {
+  await ensureHelia();
+
+  if (!outFile) {
+    const base = path.basename(filePath, path.extname(filePath));
+    outFile = `${base}.cbyte`;
+  }
+
+  const data = fs.readFileSync(filePath);
+  const chunksMap = {};
+
+  for (let offset = 0; offset < data.length; offset += chunkSize) {
+    const chunk = data.slice(offset, offset + chunkSize);
+    const hashVal = crypto.createHash('sha256').update(chunk).digest();
+    const hash = Buffer.from(hashVal).toString('hex');
+    if (!chunksMap[hash]) chunksMap[hash] = { chunk, count: 0 };
+    chunksMap[hash].count++;
+  }
+
+  const manifest = {
+    format: 'cbyte',
+    version: 1,
+    chunk_size: chunkSize,
+    original_size: data.length,
+    original_ext: path.extname(filePath),
+    chunks: []
+  };
+
+  const hashes = Object.keys(chunksMap);
+  let uploaded = 0;
+
+  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
+  for (let i = 0; i < hashes.length; i += concurrency) {
+    const batch = hashes.slice(i, i + concurrency);
+    await Promise.all(batch.map(async (h) => {
+      const info = chunksMap[h];
+
+      // Create a raw block; multiformats block API
+      const block = await Block.encode({
+        value: info.chunk,
+        codec: rawCodec,
+        hasher: sha256.sha256
+      });
+
+      await helia.blockstore.put(block.cid, block.bytes);
+
+      manifest.chunks.push({
+        cid: block.cid.toString(),
+        length: info.chunk.length,
+        count: info.count
+      });
+
+      uploaded++;
+      renderProgress(uploaded, hashes.length, 'Packing');
+    }));
+  }
+  process.stderr.write('\n');
+
+  fs.writeFileSync(outFile, JSON.stringify(manifest, null, 2));
+  console.log(`Packed ${filePath} -> ${outFile}`);
+}
+
+// Unpack using Helia
+async function unpack(manifestPath, outFile) {
+  await ensureHelia();
+
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  if (!outFile) {
+    const base = path.basename(manifestPath, path.extname(manifestPath));
+    const ext = manifest.original_ext || '.out';
+    outFile = `${base}${ext}`;
+  }
+
+  const fd = fs.openSync(outFile, 'w');
+  const total = manifest.chunks.reduce((sum, e) => sum + e.count, 0);
+  let completed = 0;
+
+  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
+  for (let i = 0; i < manifest.chunks.length; i += concurrency) {
+    const batch = manifest.chunks.slice(i, i + concurrency);
+    await Promise.all(batch.map(async (entry) => {
+      const blockBytes = await helia.blockstore.get(entry.cid);
+      for (let j = 0; j < entry.count; j++) {
+        fs.writeSync(fd, blockBytes.slice(0, entry.length));
+        completed++;
+        renderProgress(completed, total, 'Unpacking');
+      }
+    }));
+  }
+  process.stderr.write('\n');
+
+  fs.ftruncateSync(fd, manifest.original_size);
+  fs.closeSync(fd);
+  console.log(`Unpacked -> ${outFile}`);
+}
+
+// Verify blocks exist in Helia
+async function verify(manifestPath) {
+  await ensureHelia();
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  let checked = 0;
+  for (const { cid } of manifest.chunks) {
+    try {
+      await helia.blockstore.get(cid);
+      checked++;
+      renderProgress(checked, manifest.chunks.length, 'Verifying');
+    } catch {
+      console.error(`Missing block ${cid}`);
+      process.exit(1);
+    }
+  }
+  process.stderr.write('\n');
+  console.log('All blocks verified');
+}
+
+// Pin via Helia’s pin API
+async function pin(manifestPath) {
+  await ensureHelia();
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  let pinned = 0;
+  for (const { cid } of manifest.chunks) {
+    await helia.pin.add(cid);
+    pinned++;
+    renderProgress(pinned, manifest.chunks.length, 'Pinning');
+  }
+  process.stderr.write('\n');
+  console.log('All blocks pinned');
+}
+
+// CLI definition
+program.version('1.0.0');
+
+program
+  .command('pack <file>')
+  .option('-o, --output <file>', 'Output manifest')
+  .option('--chunk-size <bytes>', 'Chunk size', '4096')
+  .option('--pin', 'Pin blocks after pack', false)
+  .action((file, opts) => pack(file, opts.output, parseInt(opts['chunkSize']), opts.pin));
+
+program
+  .command('unpack <manifest>')
+  .option('-o, --output <file>', 'Output file')
+  .action((m, opts) => unpack(m, opts.output));
+
+program
+  .command('verify <manifest>')
+  .action((m) => verify(m));
+
+program
+  .command('pin <manifest>')
+  .action((m) => pin(m));
+
+program.parse(process.argv);
+

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "cbyte",
-  "version": "1.4.0",
+  "type": "module",
+  "version": "1.5.0",
   "description": "Content-addressed deduplicated storage CLI using IPFS with automatic unpack extension retention",
-  "main": "cbyte.js",
+  "main": "cbyte.cjs",
   "bin": {
-    "cbyte": "./cbyte.js"
+    "cbyte": "./cbyte.cjs"
   },
   "scripts": {
-    "test": "node test.js"
+    "test": "node test.cjs"
   },
   "keywords": [
     "ipfs",
@@ -16,13 +17,15 @@
     "storage",
     "content-addressed"
   ],
-  "author": "Your Name <you@example.com>",
+  "author": "Coperbyte <coperbyte@coperbyte.com>",
   "license": "MIT",
   "dependencies": {
+    "@helia/unixfs": "^7.0.1",
     "commander": "^11.0.0",
     "form-data": "^4.0.0",
-    "undici": "^6.0.0",
-    "ipfs-core": "^0.18.1"
+    "helia": "^6.0.16",
+    "multiformats": "^13.4.2",
+    "undici": "^6.0.0"
   },
   "engines": {
     "node": ">=20"

package/cbyte.js

@@ -1,189 +0,0 @@
-#!/usr/bin/env node
-
-require('events').defaultMaxListeners = 6000;
-
-const fs = require('fs');
-const crypto = require('crypto');
-const { program } = require('commander');
-const { create } = require('ipfs-core');
-const os = require('os');
-const path = require('path');
-
-let ipfs = null;
-
-// ------------------- IPFS Initialization -------------------
-async function ensureIpfs() {
-  if (!ipfs) {
-    console.log('Starting embedded IPFS node...');
-    ipfs = await create({ repo: path.join(process.cwd(), '.ipfs') });
-    console.log('IPFS node started.');
-  }
-}
-
-// ------------------- Utility Functions -------------------
-function sha256(data) {
-  return crypto.createHash('sha256').update(data).digest('hex');
-}
-
-function renderProgress(completed, total, label = '') {
-  const width = 40;
-  const ratio = completed / total;
-  const filled = Math.floor(ratio * width);
-  const bar = '='.repeat(filled) + ' '.repeat(width - filled);
-  process.stderr.write(`\r${label} [${bar}] ${(ratio*100).toFixed(1)}% (${completed}/${total})`);
-}
-
-// ------------------- Pack -------------------
-async function pack(filePath, output, chunkSize = 4096, pin = false) {
-  await ensureIpfs();
-
-  if (!output) {
-    const base = path.basename(filePath, path.extname(filePath));
-    output = `${base}.cbyte`;
-  }
-
-  const data = fs.readFileSync(filePath);
-  const chunksMap = {};
-
-  for (let offset = 0; offset < data.length; offset += chunkSize) {
-    const chunk = data.slice(offset, offset + chunkSize);
-    const hash = sha256(chunk);
-    if (!chunksMap[hash]) chunksMap[hash] = { chunk, count: 0 };
-    chunksMap[hash].count++;
-  }
-
-  const manifest = {
-    format: 'cbyte',
-    version: 1,
-    chunk_size: chunkSize,
-    original_size: data.length,
-    original_ext: path.extname(filePath),
-    chunks: []
-  };
-
-  const hashes = Object.keys(chunksMap);
-  const total = hashes.length;
-  let completed = 0;
-
-  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
-  for (let i = 0; i < hashes.length; i += concurrency) {
-    const batch = hashes.slice(i, i + concurrency);
-    await Promise.all(batch.map(async (hash) => {
-      const info = chunksMap[hash];
-      const { cid } = await ipfs.block.put(info.chunk);
-      if (pin) await ipfs.pin.add(cid);
-      manifest.chunks.push({ cid: cid.toString(), length: info.chunk.length, count: info.count });
-      completed++;
-      renderProgress(completed, total, 'Packing');
-    }));
-  }
-  process.stderr.write('\n');
-
-  fs.writeFileSync(output, JSON.stringify(manifest, null, 2));
-  console.log(`Packed ${filePath} -> ${output}`);
-}
-
-// ------------------- Unpack -------------------
-async function unpack(manifestPath, output) {
-  await ensureIpfs();
-
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  if (!output) {
-    const base = path.basename(manifestPath, path.extname(manifestPath));
-    const ext = manifest.original_ext || '.out';
-    output = `${base}${ext}`;
-  }
-
-  const outFd = fs.openSync(output, 'w');
-  const totalChunks = manifest.chunks.reduce((a, c) => a + c.count, 0);
-  let completed = 0;
-
-  const concurrency = Math.max(1, Math.min(8, os.cpus().length));
-  for (let i = 0; i < manifest.chunks.length; i += concurrency) {
-    const batch = manifest.chunks.slice(i, i + concurrency);
-    await Promise.all(batch.map(async (entry) => {
-      const data = Buffer.from(await ipfs.block.get(entry.cid));
-      for (let j = 0; j < entry.count; j++) {
-        fs.writeSync(outFd, data.slice(0, entry.length));
-        completed++;
-        renderProgress(completed, totalChunks, 'Unpacking');
-      }
-    }));
-  }
-
-  process.stderr.write('\n');
-  fs.ftruncateSync(outFd, manifest.original_size);
-  fs.closeSync(outFd);
-  console.log(`Unpacked -> ${output}`);
-}
-
-// ------------------- Verify -------------------
-async function verify(manifestPath) {
-  await ensureIpfs();
-
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  let verified = 0;
-
-  for (const { cid } of manifest.chunks) {
-    try {
-      await ipfs.block.get(cid);
-      verified++;
-      renderProgress(verified, manifest.chunks.length, 'Verifying');
-    } catch {
-      console.error(`Missing block ${cid}`);
-      process.exit(1);
-    }
-  }
-  process.stderr.write('\n');
-  console.log('All blocks verified');
-}
-
-// ------------------- Pin -------------------
-async function pin(manifestPath) {
-  await ensureIpfs();
-
-  const manifest = JSON.parse(fs.readFileSync(manifestPath));
-  let pinned = 0;
-
-  for (const { cid } of manifest.chunks) {
-    await ipfs.pin.add(cid);
-    pinned++;
-    renderProgress(pinned, manifest.chunks.length, 'Pinning');
-  }
-  process.stderr.write('\n');
-  console.log('All blocks pinned');
-}
-
-// ------------------- CLI -------------------
-program.version('1.0.0');
-
-program
-  .command('pack <file>')
-  .option('-o, --output <file>', 'Output manifest')
-  .option('--chunk-size <bytes>', 'Chunk size', '4096')
-  .option('--pin', 'Pin blocks after upload', false)
-  .action(async (file, options) => {
-    await pack(file, options.output, parseInt(options.chunkSize), options.pin);
-  });
-
-program
-  .command('unpack <manifest>')
-  .option('-o, --output <file>', 'Output file')
-  .action(async (manifest, options) => {
-    await unpack(manifest, options.output);
-  });
-
-program
-  .command('verify <manifest>')
-  .action(async (manifest) => {
-    await verify(manifest);
-  });
-
-program
-  .command('pin <manifest>')
-  .action(async (manifest) => {
-    await pin(manifest);
-  });
-
-program.parse(process.argv);
-